diff --git "a/None/verifier-reproduce/wandb/debug-internal.log" "b/None/verifier-reproduce/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/None/verifier-reproduce/wandb/debug-internal.log" @@ -0,0 +1,5458 @@ +2024-05-07 17:25:51,804 INFO StreamThr :125735 [internal.py:wandb_internal():88] W&B internal server running at pid: 125735, started at: 2024-05-07 17:25:51.804213 +2024-05-07 17:25:51,806 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: status +2024-05-07 17:25:51,841 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: status +2024-05-07 17:25:51,842 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: header +2024-05-07 17:25:51,842 INFO WriterThread:125735 [datastore.py:open_for_write():77] open: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/run-itlu7dts.wandb +2024-05-07 17:25:52,383 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: run +2024-05-07 17:25:52,384 INFO SenderThread:125735 [tracking_sender.py:_maybe_setup_resume():588] checking resume status for None/verifier-reproduce/itlu7dts +2024-05-07 17:25:53,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: check_version +2024-05-07 17:25:53,309 INFO SenderThread:125735 [dir_watcher.py:__init__():216] watching files in: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files +2024-05-07 17:25:53,309 INFO SenderThread:125735 [tracking_sender.py:_start_run_threads():934] run started: itlu7dts with start time 1715073951.802305 +2024-05-07 17:25:53,310 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:25:53,310 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:25:53,310 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: check_version +2024-05-07 17:25:54,310 INFO Thread-15 :125735 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:25:58,312 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: run_start +2024-05-07 17:25:58,354 DEBUG HandlerThread:125735 [system_info.py:__init__():31] System info init +2024-05-07 17:25:58,354 DEBUG HandlerThread:125735 [system_info.py:__init__():46] System info init done +2024-05-07 17:25:58,354 INFO HandlerThread:125735 [system_monitor.py:start():150] Starting system monitor +2024-05-07 17:25:58,355 INFO SystemMonitor:125735 [system_monitor.py:_start():116] Starting system asset monitoring threads +2024-05-07 17:25:58,355 INFO HandlerThread:125735 [system_monitor.py:probe():168] Collecting system info +2024-05-07 17:25:58,355 INFO SystemMonitor:125735 [interfaces.py:start():168] Started cpu +2024-05-07 17:25:58,355 INFO SystemMonitor:125735 [interfaces.py:start():168] Started disk +2024-05-07 17:25:58,356 INFO SystemMonitor:125735 [interfaces.py:start():168] Started gpu +2024-05-07 17:25:58,356 INFO SystemMonitor:125735 [interfaces.py:start():168] Started memory +2024-05-07 17:25:58,357 INFO SystemMonitor:125735 [interfaces.py:start():168] Started network +2024-05-07 17:25:58,411 DEBUG HandlerThread:125735 [system_info.py:probe():195] Probing system +2024-05-07 17:25:58,418 DEBUG HandlerThread:125735 [system_info.py:_probe_git():180] Probing git +2024-05-07 17:25:58,432 DEBUG HandlerThread:125735 [system_info.py:_probe_git():188] Probing git done +2024-05-07 17:25:58,432 DEBUG HandlerThread:125735 [system_info.py:probe():241] Probing system done +2024-05-07 17:25:58,432 DEBUG HandlerThread:125735 [system_monitor.py:probe():177] {'os': 'Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31', 'python': '3.9.2', 'heartbeatAt': '2024-05-07T09:25:58.411901', 'startedAt': '2024-05-07T09:25:51.794582', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path', '/opt/tiger/models/lean4_random_15k_all/checkpoint-236/', '--data_dir', '../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl', '--data_id', 'lean4test', '--target_set', 'train', '--save_dir', '/opt/tiger/models/lean4_random_15k_all-sample10-psv-0.5', '--generator_id', 'lean4_random_15k_all', '--verifier_id', 'psv-0.5', '--dedup', 'True', '--per_problem_sampling_solution', '10', '--loss_level', 'token', '--loss_on_llm', 'True', '--num_train_epoches', '1', '--eval_steps', '10000', '--per_device_train_batch_size', '16', '--per_device_eval_batch_size', '64', '--gradient_accumulation_steps', '4', '--gradient_checkpointing', 'True', '--learning_rate', '2e-6', '--weight_decay', '0', '--save_steps', '1000', '--lr_scheduler_type', 'linear', '--warmup_steps', '0', '--save_epoches', '1', '--save_best', 'False', '--save_total_limit', '0', '--logging_dir', 'None', '--logging_steps', '1', '--easy', 'True', '--process', 'True', '--seed', '42'), 'state': 'running', 'program': '/opt/tiger/mariana/auto-info/PSV/train_psv.py', 'codePath': 'auto-info/PSV/train_psv.py', 'git': {'remote': 'git@code.byted.org:seed/mariana.git', 'commit': 'ce8cbcbc8313f5085e52656d8bbf2e60e3c0d785'}, 'email': 'lujianqiao@bytedance.com', 'root': '/opt/tiger/mariana', 'host': 'n122-143-013', 'username': 'tiger', 'executable': '/usr/bin/python3', 'cpu_count': 90, 'cpu_count_logical': 180, 'cpu_freq': {'current': 2600.0250000000083, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}, {'current': 2600.025, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 3519.748878479004, 'used': 299.1645317077637}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2932.9151611328125}} +2024-05-07 17:25:58,433 INFO HandlerThread:125735 [system_monitor.py:probe():178] Finished collecting system info +2024-05-07 17:25:58,433 INFO HandlerThread:125735 [system_monitor.py:probe():181] Publishing system info +2024-05-07 17:25:58,433 DEBUG HandlerThread:125735 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2024-05-07 17:25:58,433 DEBUG HandlerThread:125735 [system_info.py:_save_pip():67] Saving pip packages done +2024-05-07 17:25:58,434 INFO HandlerThread:125735 [system_monitor.py:probe():183] Finished publishing system info +2024-05-07 17:25:58,469 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: files +2024-05-07 17:25:58,469 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-metadata.json with policy now +2024-05-07 17:25:58,472 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:25:58,472 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:25:58,472 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: telemetry +2024-05-07 17:25:58,529 INFO Thread-17 :125735 [upload_job.py:push():143] Uploaded file /tmp/tmp3my2r6edwandb/1fhba7c1-wandb-metadata.json +2024-05-07 17:25:59,311 INFO Thread-15 :125735 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:25:59,311 INFO Thread-15 :125735 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-metadata.json +2024-05-07 17:25:59,311 INFO Thread-15 :125735 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/requirements.txt +2024-05-07 17:26:01,312 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:26:13,475 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:26:13,476 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:26:21,317 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:26:22,317 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/config.yaml +2024-05-07 17:26:28,478 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:26:28,478 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:26:40,548 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:26:41,322 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:26:43,323 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:26:43,480 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:26:43,480 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:26:53,680 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:26:53,681 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:26:53,681 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:26:53,683 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:26:54,326 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:26:55,326 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:26:58,357 DEBUG SystemMonitor:125735 [system_monitor.py:_start():130] Starting system metrics aggregation loop +2024-05-07 17:26:58,359 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:26:58,482 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:26:58,482 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:27:07,646 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:27:07,646 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:27:07,646 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:27:07,647 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:27:08,330 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:27:09,330 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:27:13,484 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:27:13,484 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:27:27,336 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:27:28,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:27:28,486 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:27:28,486 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:27:43,488 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:27:43,488 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:27:48,737 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:27:48,737 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:27:48,738 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:27:48,738 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:27:49,341 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:27:51,342 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:27:58,363 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:27:58,490 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:27:58,490 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:27:58,570 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:27:58,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:27:58,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:27:58,572 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:27:59,344 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:27:59,344 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:28:13,492 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:28:13,492 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:28:20,948 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:28:20,949 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:28:20,949 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:28:20,950 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:28:21,350 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:28:23,350 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:28:28,365 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:28:28,493 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:28:28,494 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:28:33,353 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:28:40,502 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:28:40,503 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:28:40,503 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:28:40,503 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:28:41,355 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:28:41,355 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:28:43,495 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:28:43,496 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:28:53,745 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:28:53,745 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:28:53,746 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:28:53,746 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:28:54,358 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:28:55,359 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:28:58,367 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:28:58,497 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:28:58,498 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:29:13,500 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:29:13,500 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:29:16,010 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:29:16,011 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:29:16,011 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:29:16,012 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:29:16,364 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:29:17,364 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:29:28,368 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:29:28,502 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:29:28,502 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:29:31,369 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:29:37,792 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:29:37,792 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:29:37,793 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:29:37,793 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:29:38,371 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:29:39,371 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:29:43,503 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:29:43,504 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:29:52,499 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:29:52,499 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:29:52,500 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:29:52,500 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:29:53,377 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:29:53,377 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:29:58,370 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:29:58,505 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:29:58,506 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:30:13,507 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:30:13,508 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:30:15,514 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:30:15,514 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:30:15,515 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:30:15,515 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:30:16,383 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:30:17,383 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:30:28,372 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:30:28,509 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:30:28,509 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:30:39,389 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:30:43,511 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:30:43,511 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:30:58,373 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:30:58,513 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:30:58,513 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:30:59,878 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:30:59,878 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:30:59,879 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:30:59,879 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:31:00,394 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:31:01,395 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:31:13,515 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:31:13,516 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:31:21,446 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:31:21,446 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:31:21,447 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:31:21,447 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:31:22,400 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:31:23,400 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:31:28,375 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:31:28,517 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:31:28,517 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:31:32,134 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:31:32,134 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:31:32,135 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:31:32,135 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:31:32,403 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:31:33,403 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:31:43,406 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:31:43,519 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:31:43,520 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:31:54,834 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:31:54,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:31:54,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:31:54,835 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:31:55,409 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:31:55,409 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:31:58,377 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:31:58,522 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:31:58,522 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:32:04,037 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:32:04,037 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:32:04,037 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:32:04,038 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:32:04,411 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:32:05,412 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:32:13,524 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:32:13,524 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:32:15,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:32:15,307 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:32:15,307 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:32:15,308 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:32:15,414 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:32:17,415 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:32:28,379 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:32:28,526 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:32:28,526 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:32:29,418 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:32:38,122 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:32:38,122 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:32:38,122 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:32:38,123 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:32:38,420 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:32:39,421 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:32:43,528 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:32:43,528 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:32:48,490 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:32:48,490 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:32:48,491 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:32:48,491 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:32:49,423 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:32:49,424 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:32:58,380 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:32:58,530 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:32:58,530 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:33:00,994 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:33:00,994 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:33:00,995 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:33:00,995 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:33:01,427 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:33:03,428 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:33:11,430 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:33:13,532 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:33:13,532 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:33:20,673 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:33:20,673 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:33:20,674 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:33:20,674 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:33:21,433 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:33:21,433 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:33:28,382 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:33:28,534 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:33:28,534 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:33:43,262 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:33:43,262 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:33:43,263 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:33:43,263 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:33:43,439 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:33:43,535 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:33:43,536 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:33:45,439 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:33:52,033 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:33:52,033 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:33:52,033 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:33:52,034 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:33:52,441 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:33:53,441 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:33:58,384 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:33:58,537 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:33:58,537 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:34:03,444 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:34:13,443 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:34:13,444 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:34:13,444 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:34:13,445 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:34:13,447 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:34:13,539 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:34:13,539 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:34:15,447 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:34:28,386 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:34:28,541 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:34:28,541 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:34:35,467 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:34:35,467 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:34:35,468 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:34:35,468 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:34:36,453 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:34:37,453 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:34:43,543 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:34:43,543 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:34:44,664 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:34:44,665 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:34:44,665 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:34:44,666 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:34:45,455 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:34:45,456 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:34:55,458 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:34:58,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:34:58,545 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:34:58,545 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:35:13,547 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:35:13,548 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:35:18,426 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:35:18,427 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:35:18,427 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:35:18,428 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:35:18,465 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:35:19,465 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:35:28,389 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:35:28,549 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:35:28,549 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:35:29,001 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:35:29,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:35:29,002 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:35:29,002 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:35:29,468 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:35:29,468 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:35:39,903 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:35:39,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:35:39,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:35:39,905 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:35:40,471 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:35:41,471 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:35:43,551 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:35:43,551 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:35:58,391 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:35:58,553 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:35:58,553 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:36:03,477 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:36:13,294 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:36:13,294 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:36:13,294 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:36:13,295 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:36:13,480 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:36:13,555 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:36:13,555 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:36:15,480 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:36:27,538 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:36:27,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:36:27,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:36:27,540 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:36:28,392 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:36:28,483 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:36:28,556 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:36:28,557 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:36:29,484 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:36:43,558 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:36:43,558 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:36:43,953 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:36:43,953 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:36:43,954 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:36:43,954 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:36:44,487 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:36:45,488 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:36:53,490 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:36:58,394 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:36:58,560 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:36:58,560 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:37:04,740 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:37:04,740 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:37:04,740 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:37:04,741 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:37:05,493 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:37:05,493 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:37:13,562 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:37:13,562 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:37:16,557 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:37:16,557 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:37:16,557 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:37:16,558 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:37:17,496 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:37:17,496 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:37:28,396 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:37:28,564 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:37:28,564 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:37:38,739 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:37:38,739 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:37:38,739 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:37:38,740 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:37:39,501 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:37:39,502 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:37:43,566 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:37:43,566 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:37:53,505 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:37:58,397 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:37:58,568 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:37:58,568 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:38:07,963 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:38:07,963 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:38:07,964 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:38:07,964 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:38:08,509 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:38:09,509 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:38:13,570 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:38:13,570 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:38:17,976 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:38:17,977 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:38:17,977 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:38:17,978 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:38:18,512 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:38:19,512 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:38:28,399 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:38:28,571 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:38:28,572 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:38:39,932 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:38:39,932 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:38:39,933 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:38:39,933 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:38:40,517 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:38:41,518 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:38:43,573 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:38:43,574 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:38:49,520 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:38:58,401 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:38:58,575 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:38:58,575 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:39:11,853 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:39:11,853 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:39:11,854 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:39:11,855 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:39:12,526 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:39:13,526 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:39:13,577 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:39:13,577 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:39:28,402 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:39:28,579 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:39:28,579 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:39:35,277 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:39:35,277 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:39:35,278 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:39:35,278 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:39:35,532 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:39:35,532 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:39:43,581 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:39:43,581 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:39:44,914 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:39:44,914 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:39:44,915 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:39:44,915 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:39:45,534 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:39:45,534 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:39:58,404 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:39:58,583 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:39:58,583 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:39:59,538 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:40:13,433 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:40:13,433 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:40:13,433 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:40:13,434 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:40:13,541 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:40:13,584 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:40:13,585 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:40:15,542 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:40:28,406 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:40:28,586 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:40:28,586 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:40:34,170 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:40:34,171 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:40:34,171 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:40:34,172 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:40:34,548 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:40:35,548 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:40:43,588 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:40:43,588 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:40:45,387 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:40:45,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:40:45,388 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:40:45,388 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:40:45,551 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:40:45,551 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:40:58,407 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:40:58,590 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:40:58,590 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:41:01,556 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:41:03,556 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:41:13,592 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:41:13,592 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:41:16,791 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:41:16,792 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:41:16,792 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:41:16,793 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:41:17,560 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:41:17,560 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:41:28,409 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:41:28,594 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:41:28,594 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:41:38,597 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:41:38,597 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:41:38,597 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:41:38,598 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:41:39,566 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:41:39,566 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:41:43,595 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:41:43,596 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:41:58,411 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:41:58,597 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:41:58,597 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:42:00,510 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:42:00,511 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:42:00,511 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:42:00,511 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:42:00,571 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:42:01,571 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:42:13,599 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:42:13,599 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:42:23,576 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:42:28,412 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:42:28,601 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:42:28,601 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:42:42,573 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:42:42,574 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:42:42,574 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:42:42,575 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:42:42,581 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:42:43,581 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:42:43,602 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:42:43,603 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:42:58,414 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:42:58,604 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:42:58,605 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:43:06,104 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:43:06,104 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:43:06,104 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:43:06,105 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:43:06,588 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:43:08,588 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:43:13,607 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:43:13,607 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:43:16,770 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:43:16,770 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:43:16,770 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:43:16,771 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:43:17,590 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:43:18,591 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:43:28,416 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:43:28,593 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:43:28,608 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:43:28,609 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:43:43,610 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:43:43,611 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:43:49,045 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:43:49,045 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:43:49,045 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:43:49,046 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:43:49,599 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:43:50,599 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:43:58,417 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:43:58,612 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:43:58,612 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:44:01,127 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:44:01,127 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:44:01,128 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:44:01,128 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:44:01,602 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:44:02,602 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:44:13,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:44:13,306 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:44:13,307 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:44:13,307 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:44:13,605 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:44:13,614 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:44:13,614 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:44:14,605 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:44:28,419 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:44:28,793 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:44:28,794 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:44:30,610 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:44:37,609 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:44:37,609 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:44:37,610 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:44:37,610 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:44:37,611 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:44:38,612 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:44:43,795 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:44:43,796 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:44:49,154 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:44:49,154 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:44:49,154 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:44:49,155 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:44:49,614 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:44:50,615 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:44:58,421 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:44:58,797 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:44:58,797 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:44:59,001 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:44:59,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:44:59,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:44:59,002 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:44:59,617 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:45:00,618 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:45:10,620 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:45:13,799 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:45:13,799 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:45:20,332 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:45:20,332 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:45:20,332 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:45:20,333 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:45:20,623 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:45:22,623 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:45:28,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:45:28,801 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:45:28,801 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:45:29,047 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:45:29,048 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:45:29,048 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:45:29,049 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:45:29,625 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:45:30,625 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:45:39,521 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:45:39,522 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:45:39,522 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:45:39,523 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:45:39,627 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:45:40,628 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:45:43,803 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:45:43,803 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:45:58,424 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:45:58,805 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:45:58,805 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:46:04,633 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:46:13,361 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:46:13,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:46:13,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:46:13,362 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:46:13,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:46:13,807 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:46:13,807 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:46:14,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:46:22,443 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:46:22,443 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:46:22,443 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:46:22,444 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:46:22,638 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:46:24,639 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:46:28,426 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:46:28,808 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:46:28,809 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:46:33,423 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:46:33,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:46:33,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:46:33,424 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:46:33,641 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:46:34,641 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:46:43,810 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:46:43,810 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:46:56,646 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:46:58,427 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:46:58,812 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:46:58,812 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:47:06,208 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:47:06,208 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:47:06,208 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:47:06,209 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:47:06,649 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:47:08,649 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:47:13,814 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:47:13,814 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:47:16,388 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:47:16,388 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:47:16,388 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:47:16,389 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:47:16,651 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:47:18,652 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:47:28,429 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:47:28,816 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:47:28,816 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:47:38,074 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:47:38,074 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:47:38,075 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:47:38,075 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:47:38,657 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:47:40,658 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:47:43,818 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:47:43,818 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:47:48,660 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:47:58,431 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:47:58,820 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:47:58,820 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:48:10,120 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:48:10,120 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:48:10,121 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:48:10,121 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:48:10,666 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:48:12,667 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:48:13,822 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:48:13,822 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:48:28,433 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:48:28,824 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:48:28,824 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:48:33,017 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:48:33,017 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:48:33,018 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:48:33,018 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:48:33,672 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:48:34,672 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:48:43,826 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:48:43,826 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:48:54,422 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:48:54,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:48:54,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:48:54,423 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:48:54,677 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:48:56,677 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:48:58,434 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:48:58,828 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:48:58,828 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:49:04,679 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:49:13,830 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:49:13,830 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:49:14,204 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:49:14,204 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:49:14,204 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:49:14,205 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:49:14,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:49:16,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:49:27,620 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:49:27,620 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:49:27,621 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:49:27,621 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:49:27,685 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:49:28,436 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:49:28,685 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:49:28,832 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:49:28,832 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:49:36,409 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:49:36,409 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:49:36,409 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:49:36,410 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:49:36,688 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:49:38,688 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:49:43,833 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:49:43,834 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:49:48,691 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:49:58,437 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:49:58,835 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:49:58,835 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:50:07,484 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:50:07,484 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:50:07,485 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:50:07,485 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:50:07,695 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:50:08,696 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:50:13,837 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:50:13,837 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:50:28,439 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:50:28,839 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:50:28,839 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:50:29,298 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:50:29,298 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:50:29,298 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:50:29,299 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:50:29,701 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:50:30,701 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:50:43,841 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:50:43,841 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:50:51,913 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:50:51,913 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:50:51,913 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:50:51,914 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:50:52,706 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:50:52,706 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:50:58,440 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:50:58,843 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:50:58,843 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:51:13,845 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:51:13,845 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:51:14,712 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:51:28,442 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:51:28,847 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:51:28,847 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:51:35,361 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:51:35,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:51:35,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:51:35,362 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:51:35,717 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:51:36,717 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:51:43,848 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:51:43,849 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:51:50,926 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:51:50,926 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:51:50,926 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:51:50,927 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:51:51,721 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:51:52,721 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:51:58,444 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:51:58,850 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:51:58,851 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:51:59,498 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:51:59,499 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:51:59,499 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:51:59,500 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:51:59,723 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:52:00,723 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:52:12,726 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:52:13,852 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:52:13,853 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:52:20,690 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:52:20,690 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:52:20,690 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:52:20,691 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:52:20,728 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:52:22,729 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:52:28,445 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:52:28,854 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:52:28,854 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:52:31,455 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:52:31,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:52:31,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:52:31,456 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:52:31,731 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:52:32,731 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:52:43,035 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:52:43,035 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:52:43,035 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:52:43,036 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:52:43,734 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:52:43,856 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:52:43,856 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:52:44,734 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:52:58,447 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:52:58,858 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:52:58,858 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:53:06,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:08,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:13,860 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:53:13,860 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:53:16,243 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:53:16,243 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:53:16,243 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:53:16,244 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:53:16,742 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:16,743 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:53:18,743 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:28,449 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:53:28,861 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:53:28,862 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:53:37,549 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:53:37,549 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:53:37,550 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:53:37,550 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:53:37,748 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:53:38,748 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:43,863 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:53:43,864 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:53:47,963 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:53:47,963 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:53:47,964 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:53:47,964 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:53:48,751 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:48,751 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:53:58,450 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:53:58,753 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:53:58,866 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:53:58,866 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:54:06,588 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:54:06,588 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:54:06,588 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:54:06,589 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:54:06,755 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:54:08,756 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:54:13,868 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:54:13,868 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:54:17,457 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:54:17,457 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:54:17,457 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:54:17,458 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:54:17,758 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:54:18,758 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:54:28,452 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:54:28,869 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:54:28,869 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:54:38,335 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:54:38,335 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:54:38,335 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:54:38,336 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:54:38,763 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:54:40,764 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:54:43,871 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:54:43,871 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:54:52,767 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:54:58,454 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:54:58,873 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:54:58,873 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:54:59,170 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:54:59,171 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:54:59,171 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:54:59,172 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:54:59,769 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:55:00,769 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:08,361 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:55:08,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:55:08,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:55:08,362 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:55:08,771 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:08,771 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:55:10,771 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:13,875 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:55:13,875 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:55:18,556 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:55:18,557 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:55:18,557 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:55:18,558 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:55:18,773 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:55:20,774 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:28,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:55:28,776 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:28,876 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:55:28,877 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:55:42,890 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:55:42,890 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:55:42,891 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:55:42,891 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:55:43,780 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:55:43,878 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:55:43,878 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:55:44,780 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:54,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:55:54,419 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:55:54,419 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:55:54,419 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:55:54,783 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:55:56,783 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:55:58,457 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:55:58,880 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:55:58,880 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:56:04,044 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:56:04,044 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:56:04,044 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:56:04,045 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:56:04,785 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:56:04,785 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:56:13,882 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:56:13,882 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:56:28,459 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:56:28,791 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:56:28,884 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:56:28,884 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:56:43,886 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:56:43,886 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:56:49,098 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:56:49,098 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:56:49,099 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:56:49,099 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:56:49,796 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:56:50,797 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:56:58,461 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:56:58,888 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:56:58,888 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:57:00,486 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:57:00,486 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:57:00,487 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:57:00,487 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:57:00,799 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:57:02,800 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:57:10,706 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:57:10,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:57:10,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:57:10,707 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:57:10,802 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:57:12,802 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:57:13,890 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:57:13,890 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:57:28,462 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:57:28,892 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:57:28,892 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:57:30,807 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:57:43,893 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:57:43,894 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:57:52,583 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:57:52,583 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:57:52,584 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:57:52,584 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:57:52,812 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:57:54,813 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:57:58,464 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:57:58,896 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:57:58,896 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:58:13,898 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:58:13,898 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:58:14,894 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:58:14,894 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:58:14,895 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:58:14,895 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:58:15,818 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:58:16,819 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:58:26,214 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:58:26,214 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:58:26,214 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:58:26,215 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:58:26,821 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:58:26,821 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:58:28,466 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:58:28,900 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:58:28,900 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:58:43,902 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:58:43,902 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:58:50,827 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:58:58,468 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:58:58,904 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:58:58,904 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:58:59,841 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:58:59,841 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:58:59,842 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:58:59,842 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:59:00,830 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:59:00,830 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:59:11,178 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:59:11,178 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:59:11,178 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:59:11,179 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:59:11,833 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:59:12,833 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:59:13,906 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:59:13,906 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:59:19,942 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:59:19,942 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:59:19,942 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:59:19,943 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:59:20,835 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:59:20,835 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:59:28,469 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:59:28,908 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:59:28,908 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:59:43,910 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:59:43,910 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 17:59:44,841 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:59:56,753 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 17:59:56,753 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 17:59:56,753 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 17:59:56,754 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 17:59:56,844 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 17:59:58,470 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 17:59:58,845 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 17:59:58,912 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 17:59:58,912 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:00:08,592 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:00:08,592 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:00:08,593 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:00:08,593 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:00:08,848 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:00:08,848 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:00:13,914 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:00:13,914 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:00:20,878 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:00:20,879 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:00:20,879 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:00:20,880 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:00:21,851 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:00:22,851 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:00:28,472 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:00:28,916 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:00:28,916 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:00:43,917 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:00:43,918 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:00:44,857 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:00:58,474 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:00:58,919 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:00:58,920 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:01:05,564 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:01:05,564 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:01:05,564 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:01:05,565 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:01:05,863 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:01:06,863 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:01:13,921 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:01:13,921 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:01:26,289 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:01:26,289 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:01:26,290 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:01:26,290 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:01:26,868 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:01:26,869 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:01:28,476 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:01:28,923 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:01:28,923 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:01:41,762 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:01:41,762 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:01:41,762 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:01:41,763 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:01:41,872 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:01:42,872 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:01:43,925 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:01:43,925 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:01:54,876 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:01:58,477 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:01:58,927 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:01:58,927 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:02:03,118 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:02:03,118 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:02:03,118 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:02:03,119 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:02:03,878 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:02:04,878 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:13,929 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:02:13,929 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:02:15,197 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:02:15,197 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:02:15,197 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:02:15,198 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:02:15,881 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:02:16,882 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:27,767 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:02:27,767 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:02:27,767 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:02:27,768 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:02:27,884 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:02:28,479 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:02:28,885 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:28,931 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:02:28,931 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:02:38,887 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:43,933 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:02:43,933 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:02:47,054 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:02:47,054 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:02:47,055 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:02:47,055 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:02:47,890 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:02:48,890 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:58,481 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:02:58,500 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:02:58,500 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:02:58,500 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:02:58,500 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:02:58,892 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:02:58,893 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:02:58,935 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:02:58,935 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:03:11,787 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:03:11,787 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:03:11,787 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:03:11,788 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:03:11,896 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:03:12,896 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:03:13,937 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:03:13,937 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:03:28,482 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:03:28,938 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:03:28,939 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:03:36,902 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:03:43,940 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:03:43,941 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:03:58,484 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:03:58,662 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:03:58,662 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:03:58,663 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:03:58,663 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:03:58,908 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:03:58,942 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:03:58,942 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:03:59,908 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:04:09,029 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:04:09,030 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:04:09,030 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:04:09,031 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:04:09,910 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:04:11,911 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:04:13,944 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:04:13,944 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:04:28,486 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:04:28,946 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:04:28,946 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:04:30,624 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:04:30,625 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:04:30,625 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:04:30,626 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:04:30,916 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:04:31,916 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:04:44,019 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:04:44,020 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:04:45,920 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:04:56,931 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:04:56,932 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:04:56,932 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:04:56,933 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:04:57,923 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:04:57,923 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:04:58,488 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:04:59,026 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:04:59,026 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:04:59,924 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:05,119 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:05:05,120 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:05:05,120 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:05:05,121 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:05:05,925 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:05:07,926 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:14,028 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:05:14,028 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:05:15,106 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:05:15,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:05:15,107 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:05:15,107 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:05:15,928 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:05:17,928 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:28,489 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:05:29,030 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:05:29,030 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:05:29,931 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:42,010 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:05:42,010 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:05:42,010 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:05:42,011 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:05:42,935 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:05:43,936 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:44,032 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:05:44,032 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:05:51,733 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:05:51,733 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:05:51,734 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:05:51,734 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:05:51,938 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:05:53,938 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:05:58,491 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:05:59,034 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:05:59,034 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:06:04,788 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:06:04,789 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:06:04,789 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:06:04,790 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:06:04,941 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:06:05,941 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:06:14,036 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:06:14,036 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:06:17,944 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:06:28,493 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:06:29,038 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:06:29,038 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:06:38,033 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:06:38,033 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:06:38,033 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:06:38,035 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:06:38,949 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:06:39,950 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:06:44,040 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:06:44,040 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:06:49,408 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:06:49,408 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:06:49,408 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:06:49,409 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:06:49,952 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:06:51,953 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:06:58,495 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:06:59,042 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:06:59,042 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:07:01,609 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:07:01,609 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:07:01,609 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:07:01,610 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:07:01,955 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:07:03,956 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:13,959 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:14,043 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:07:14,044 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:07:20,705 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:07:20,705 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:07:20,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:07:20,706 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:07:20,961 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:07:21,961 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:28,496 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:07:29,045 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:07:29,045 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:07:31,239 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:07:31,239 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:07:31,239 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:07:31,240 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:07:31,964 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:07:33,964 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:44,047 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:07:44,047 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:07:44,339 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:07:44,339 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:07:44,339 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:07:44,340 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:07:44,967 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:07:45,967 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:53,970 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:07:58,499 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:07:59,049 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:07:59,049 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:08:06,335 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:08:06,335 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:08:06,336 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:08:06,336 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:08:06,973 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:08:07,973 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:08:14,051 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:08:14,051 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:08:28,500 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:08:28,948 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:08:28,949 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:08:28,949 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:08:28,950 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:08:28,979 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:08:29,053 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:08:29,053 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:08:29,979 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:08:44,055 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:08:44,055 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:08:51,900 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:08:51,901 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:08:51,901 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:08:51,902 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:08:51,986 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:08:53,987 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:08:58,502 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:08:59,057 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:08:59,057 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:09:09,991 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:09:14,058 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:09:14,059 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:09:28,503 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:09:29,060 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:09:29,060 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:09:29,932 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:09:29,932 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:09:29,933 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:09:29,933 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:09:29,995 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:09:31,996 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:09:44,062 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:09:44,062 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:09:52,834 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:09:52,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:09:52,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:09:52,835 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:09:53,001 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:09:54,002 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:09:58,505 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:09:59,064 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:09:59,064 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:10:01,333 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:10:01,333 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:10:01,333 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:10:01,334 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:10:02,004 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:10:04,004 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:10:14,066 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:10:14,066 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:10:24,009 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:10:28,507 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:10:29,068 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:10:29,068 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:10:43,328 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:10:43,329 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:10:43,329 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:10:43,330 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:10:44,014 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:10:44,014 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:10:44,070 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:10:44,070 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:10:46,014 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:10:52,791 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:10:52,791 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:10:52,791 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:10:52,792 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:10:53,016 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:10:54,016 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:10:58,508 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:10:59,072 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:10:59,072 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:11:06,368 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:11:06,368 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:11:06,368 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:11:06,369 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:11:07,019 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:11:08,019 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:11:14,073 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:11:14,074 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:11:20,022 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:11:28,510 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:11:29,075 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:11:29,075 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:11:30,180 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:11:30,181 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:11:30,181 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:11:30,182 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:11:31,025 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:11:32,025 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:11:43,789 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:11:43,789 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:11:43,790 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:11:43,791 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:11:44,028 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:11:44,077 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:11:44,077 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:11:46,029 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:11:54,089 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:11:54,090 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:11:54,090 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:11:54,091 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:11:55,031 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:11:56,031 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:11:58,512 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:11:59,079 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:11:59,079 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:12:04,033 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:12:14,081 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:12:14,081 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:12:15,534 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:12:15,534 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:12:15,534 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:12:15,535 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:12:16,036 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:12:18,037 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:12:24,845 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:12:24,845 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:12:24,846 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:12:24,846 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:12:25,039 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:12:26,039 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:12:28,513 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:12:29,083 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:12:29,083 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:12:44,085 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:12:44,085 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:12:47,568 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:12:47,568 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:12:47,569 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:12:47,569 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:12:48,044 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:12:50,045 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:12:58,515 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:12:59,087 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:12:59,087 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:13:12,051 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:13:14,089 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:13:14,089 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:13:28,517 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:13:29,091 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:13:29,091 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:13:32,004 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:13:32,004 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:13:32,004 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:13:32,005 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:13:32,056 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:13:34,057 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:13:44,093 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:13:44,093 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:13:54,922 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:13:54,922 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:13:54,922 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:13:54,923 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:13:55,062 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:13:56,063 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:13:58,519 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:13:59,095 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:13:59,095 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:14:05,427 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:14:05,427 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:14:05,427 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:14:05,428 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:14:06,065 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:14:06,065 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:14:14,097 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:14:14,097 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:14:18,068 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:14:28,521 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:14:29,099 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:14:29,099 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:14:37,200 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:14:37,200 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:14:37,201 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:14:37,201 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:14:38,073 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:14:38,073 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:14:44,101 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:14:44,101 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:14:58,522 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:14:59,103 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:14:59,103 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:15:00,011 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:15:00,011 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:15:00,012 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:15:00,012 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:15:00,078 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:15:02,079 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:15:13,058 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:15:13,059 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:15:13,059 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:15:13,060 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:15:13,081 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:15:14,082 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:15:14,104 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:15:14,105 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:15:28,524 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:15:29,107 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:15:29,107 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:15:36,087 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:15:44,109 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:15:44,109 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:15:48,571 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:15:48,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:15:48,572 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:15:48,572 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:15:49,091 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:15:50,091 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:15:56,989 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:15:56,990 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:15:56,990 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:15:56,991 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:15:57,093 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:15:58,093 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:15:58,527 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:15:59,111 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:15:59,111 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:16:05,764 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:16:05,764 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:16:05,765 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:16:05,765 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:16:06,096 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:16:08,096 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:16:14,112 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:16:14,113 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:16:16,098 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:16:28,529 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:16:29,114 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:16:29,114 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:16:37,395 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:16:37,395 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:16:37,395 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:16:37,396 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:16:38,104 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:16:38,104 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:16:44,116 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:16:44,116 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:16:58,531 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:16:58,651 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:16:58,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:16:58,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:16:58,652 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:16:59,109 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:16:59,118 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:16:59,118 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:17:00,109 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:17:14,119 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:17:14,119 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:17:21,886 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:17:21,886 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:17:21,886 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:17:21,887 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:17:22,115 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:17:24,116 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:17:28,533 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:17:29,121 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:17:29,121 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:17:34,118 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:17:44,123 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:17:44,123 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:17:46,133 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:17:46,134 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:17:46,134 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:17:46,135 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:17:47,122 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:17:48,122 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:17:58,534 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:17:59,321 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:17:59,321 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:17:59,322 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:17:59,323 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:17:59,323 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:17:59,323 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:18:00,126 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:18:00,126 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:18:11,153 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:18:11,153 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:18:11,153 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:18:11,154 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:18:12,129 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:18:12,129 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:18:14,323 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:18:14,323 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:18:22,131 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:18:28,536 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:18:29,325 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:18:29,325 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:18:31,931 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:18:31,931 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:18:31,931 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:18:31,932 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:18:32,134 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:18:34,134 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:18:44,327 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:18:44,327 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:18:45,140 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:18:45,140 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:18:45,141 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:18:45,141 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:18:46,137 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:18:46,138 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:18:58,538 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:18:59,329 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:18:59,329 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:19:06,217 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:19:06,217 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:19:06,217 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:19:06,218 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:19:07,143 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:19:08,143 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:19:14,331 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:19:14,331 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:19:16,146 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:19:25,414 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:19:25,414 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:19:25,414 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:19:25,415 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:19:26,150 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:19:26,150 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:19:28,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:19:29,333 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:19:29,333 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:19:40,311 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:19:40,311 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:19:40,311 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:19:40,312 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:19:41,153 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:19:42,154 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:19:44,335 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:19:44,335 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:19:58,541 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:19:59,337 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:19:59,337 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:20:02,285 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:20:02,285 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:20:02,286 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:20:02,286 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:20:03,159 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:20:04,159 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:20:14,339 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:20:14,339 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:20:26,165 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:20:28,543 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:20:29,341 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:20:29,341 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:20:32,636 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:20:32,636 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:20:32,636 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:20:32,637 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:20:33,167 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:20:34,167 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:20:44,491 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:20:44,492 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:20:44,534 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:20:44,535 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:20:44,535 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:20:44,535 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:20:45,170 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:20:46,170 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:20:56,579 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:20:56,579 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:20:56,579 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:20:56,580 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:20:57,173 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:20:58,173 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:20:58,544 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:20:59,494 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:20:59,494 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:21:06,175 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:21:14,496 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:21:14,496 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:21:21,744 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:21:21,745 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:21:21,745 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:21:21,746 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:21:22,179 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:21:22,179 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:21:28,546 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:21:29,498 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:21:29,498 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:21:32,470 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:21:32,470 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:21:32,470 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:21:32,471 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:21:33,181 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:21:34,182 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:21:41,811 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:21:41,811 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:21:41,811 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:21:41,812 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:21:42,184 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:21:42,184 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:21:44,500 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:21:44,500 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:21:58,548 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:21:59,502 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:21:59,502 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:22:02,189 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:22:09,765 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:22:09,765 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:22:09,765 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:22:09,766 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:22:10,191 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:22:10,191 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:22:14,504 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:22:14,504 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:22:19,757 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:22:19,757 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:22:19,758 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:22:19,758 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:22:20,194 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:22:20,194 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:22:28,549 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:22:29,506 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:22:29,506 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:22:42,151 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:22:42,151 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:22:42,152 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:22:42,152 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:22:42,200 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:22:44,201 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:22:44,508 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:22:44,508 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:22:58,551 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:22:59,510 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:22:59,510 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:23:04,207 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:23:14,512 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:23:14,512 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:23:24,362 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:23:24,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:23:24,363 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:23:24,363 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:23:25,213 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:23:26,213 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:23:28,552 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:23:29,514 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:23:29,514 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:23:35,199 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:23:35,199 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:23:35,199 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:23:35,200 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:23:35,216 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:23:36,216 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:23:44,516 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:23:44,516 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:23:52,413 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:23:52,414 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:23:52,414 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:23:52,415 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:23:53,221 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:23:54,221 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:23:58,554 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:23:59,518 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:23:59,518 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:24:10,225 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:24:14,520 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:24:14,520 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:24:18,162 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:24:18,162 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:24:18,162 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:24:18,163 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:24:18,227 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:24:18,227 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:24:20,228 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:24:28,556 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:24:29,009 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:24:29,009 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:24:29,009 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:24:29,010 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:24:29,230 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:24:29,522 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:24:29,522 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:24:30,230 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:24:44,524 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:24:44,524 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:24:50,709 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:24:50,710 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:24:50,710 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:24:50,710 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:24:51,235 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:24:52,236 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:24:58,558 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:24:59,526 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:24:59,526 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:25:02,238 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:25:14,528 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:25:14,528 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:25:24,277 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:25:24,277 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:25:24,278 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:25:24,278 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:25:25,244 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:25:27,245 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:25:28,559 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:25:29,530 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:25:29,530 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:25:35,375 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:25:35,375 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:25:35,375 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:25:35,376 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:25:36,247 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:25:37,247 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:25:44,532 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:25:44,532 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:25:58,479 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:25:58,480 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:25:58,480 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:25:58,481 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:25:58,560 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:25:59,253 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:25:59,534 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:25:59,534 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:26:01,254 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:26:13,257 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:26:14,536 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:26:14,536 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:26:28,436 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:26:28,437 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:26:28,437 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:26:28,438 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:26:28,562 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:26:29,261 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:26:29,538 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:26:29,538 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:26:31,262 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:26:37,057 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:26:37,058 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:26:37,058 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:26:37,059 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:26:37,263 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:26:39,264 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:26:44,540 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:26:44,540 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:26:47,179 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:26:47,180 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:26:47,180 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:26:47,181 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:26:47,266 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:26:49,266 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:26:58,564 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:26:59,541 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:26:59,542 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:27:03,270 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:27:14,543 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:27:14,544 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:27:21,025 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:27:21,026 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:27:21,026 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:27:21,027 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:27:21,274 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:27:23,275 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:27:28,566 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:27:29,545 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:27:29,546 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:27:31,087 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:27:31,087 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:27:31,087 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:27:31,088 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:27:31,277 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:27:33,278 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:27:44,547 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:27:44,548 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:27:53,132 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:27:53,132 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:27:53,133 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:27:53,133 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:27:53,283 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:27:55,283 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:27:58,567 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:27:59,550 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:27:59,550 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:28:03,285 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:28:14,362 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:28:14,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:28:14,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:28:14,363 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:28:14,551 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:28:14,552 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:28:15,288 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:28:15,289 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:28:25,920 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:28:25,920 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:28:25,920 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:28:25,921 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:28:26,291 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:28:27,292 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:28:28,569 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:28:29,553 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:28:29,553 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:28:43,675 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:28:43,676 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:28:43,676 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:28:43,677 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:28:44,296 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:28:44,555 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:28:44,555 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:28:45,296 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:28:58,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:28:59,557 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:28:59,557 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:29:09,302 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:29:14,559 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:29:14,559 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:29:23,229 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:29:23,229 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:29:23,229 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:29:23,230 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:29:23,306 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:29:25,306 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:29:28,572 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:29:29,561 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:29:29,561 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:29:37,213 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:29:37,213 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:29:37,213 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:29:37,214 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:29:37,309 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:29:39,310 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:29:44,563 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:29:44,563 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:29:50,230 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:29:50,230 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:29:50,230 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:29:50,231 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:29:50,312 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:29:51,313 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:29:58,574 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:29:59,565 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:29:59,565 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:30:13,318 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:30:14,566 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:30:14,567 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:30:15,319 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:30:28,575 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:30:29,568 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:30:29,569 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:30:35,030 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:30:35,030 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:30:35,030 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:30:35,031 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:30:35,324 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:30:37,324 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:30:44,570 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:30:44,571 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:30:56,825 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:30:56,825 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:30:56,825 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:30:56,826 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:30:57,329 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:30:58,577 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:30:59,330 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:30:59,572 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:30:59,572 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:31:14,574 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:31:14,574 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:31:19,806 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:31:19,806 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:31:19,806 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:31:19,807 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:31:20,335 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:31:21,335 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:31:28,579 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:31:29,576 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:31:29,576 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:31:43,341 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:31:44,578 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:31:44,579 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:31:58,581 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:31:59,581 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:31:59,581 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:32:03,539 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:32:03,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:32:03,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:32:03,541 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:32:04,346 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:32:05,346 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:32:14,583 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:32:14,583 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:32:15,811 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:32:15,811 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:32:15,812 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:32:15,812 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:32:16,349 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:32:17,349 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:32:25,700 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:32:25,700 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:32:25,701 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:32:25,701 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:32:26,351 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:32:27,352 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:32:28,582 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:32:29,585 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:32:29,585 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:32:44,587 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:32:44,587 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:32:49,357 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:32:58,584 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:32:59,589 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:32:59,589 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:33:05,623 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:33:05,623 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:33:05,623 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:33:05,624 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:33:06,361 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:33:07,362 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:14,591 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:33:14,591 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:33:15,154 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:33:15,155 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:33:15,155 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:33:15,156 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:33:15,363 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:33:17,364 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:28,585 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:33:29,593 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:33:29,593 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:33:36,741 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:33:36,741 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:33:36,742 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:33:36,742 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:33:37,369 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:37,369 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:33:39,369 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:44,595 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:33:44,595 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:33:47,371 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:55,903 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:33:55,903 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:33:55,903 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:33:55,904 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:33:56,373 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:33:57,374 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:33:58,587 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:33:59,597 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:33:59,597 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:34:11,421 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:34:11,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:34:11,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:34:11,423 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:34:12,377 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:34:13,378 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:34:14,599 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:34:14,599 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:34:20,193 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:34:20,193 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:34:20,193 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:34:20,194 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:34:20,379 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:34:21,380 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:34:28,589 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:34:29,759 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:34:29,759 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:34:31,382 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:34:44,761 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:34:44,761 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:34:47,108 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:34:47,108 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:34:47,108 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:34:47,109 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:34:47,386 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:34:49,387 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:34:58,487 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:34:58,488 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:34:58,488 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:34:58,489 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:34:58,590 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:34:59,389 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:34:59,389 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:34:59,763 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:34:59,763 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:35:08,056 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:35:08,056 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:35:08,056 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:35:08,057 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:35:08,391 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:35:09,392 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:35:14,765 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:35:14,765 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:35:28,592 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:35:30,031 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:35:30,031 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:35:31,397 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:35:40,273 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:35:40,273 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:35:40,273 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:35:40,274 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:35:40,399 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:35:41,399 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:35:45,041 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:35:45,041 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:35:51,487 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:35:51,487 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:35:51,487 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:35:51,488 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:35:52,402 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:35:53,402 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:35:58,594 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:36:00,043 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:36:00,043 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:36:02,156 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:36:02,156 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:36:02,156 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:36:02,157 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:36:02,404 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:36:03,405 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:36:15,045 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:36:15,045 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:36:23,409 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:36:28,595 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:36:30,047 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:36:30,047 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:36:45,187 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:36:45,187 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:36:45,231 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:36:45,231 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:36:45,231 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:36:45,232 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:36:45,415 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:36:47,416 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:36:55,499 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:36:55,499 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:36:55,500 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:36:55,500 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:36:56,418 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:36:57,418 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:36:58,597 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:37:00,189 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:37:00,190 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:37:08,571 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:37:08,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:37:08,572 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:37:08,573 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:37:09,421 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:37:09,422 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:37:15,191 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:37:15,192 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:37:28,598 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:37:30,389 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:37:30,389 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:37:31,427 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:37:39,510 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:37:39,511 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:37:39,511 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:37:39,511 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:37:40,429 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:37:41,430 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:37:45,391 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:37:45,391 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:37:51,049 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:37:51,049 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:37:51,049 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:37:51,050 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:37:51,432 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:37:53,434 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:37:58,600 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:38:00,393 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:38:00,393 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:38:03,105 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:38:03,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:38:03,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:38:03,106 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:38:03,437 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:38:05,437 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:38:15,395 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:38:15,395 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:38:15,440 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:38:28,602 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:38:30,397 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:38:30,397 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:38:35,258 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:38:35,258 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:38:35,258 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:38:35,259 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:38:35,445 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:38:37,445 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:38:45,399 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:38:45,399 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:38:56,000 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:38:56,000 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:38:56,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:38:56,001 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:38:56,450 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:38:57,450 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:38:58,604 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:39:00,401 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:39:00,401 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:39:15,403 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:39:15,403 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:39:18,158 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:39:18,159 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:39:18,159 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:39:18,159 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:39:18,456 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:39:19,456 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:39:28,605 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:39:30,405 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:39:30,405 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:39:41,462 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:39:45,407 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:39:45,407 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:39:49,745 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:39:49,745 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:39:49,746 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:39:49,746 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:39:50,464 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:39:51,464 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:39:58,608 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:40:00,409 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:40:00,409 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:40:00,926 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:40:00,926 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:40:00,927 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:40:00,927 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:40:01,467 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:40:01,467 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:40:15,411 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:40:15,411 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:40:22,873 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:40:22,873 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:40:22,873 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:40:22,874 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:40:23,473 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:40:23,473 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:40:28,609 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:40:30,412 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:40:30,413 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:40:35,476 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:40:45,414 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:40:45,415 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:40:54,372 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:40:54,372 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:40:54,372 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:40:54,373 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:40:54,481 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:40:55,481 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:40:58,610 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:41:00,416 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:41:00,416 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:41:15,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:41:15,418 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:41:16,173 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:41:16,173 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:41:16,173 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:41:16,174 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:41:16,486 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:41:17,487 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:41:25,241 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:41:25,241 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:41:25,242 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:41:25,242 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:41:25,489 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:41:25,489 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:41:27,489 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:41:28,612 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:41:30,420 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:41:30,420 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:41:35,491 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:41:45,422 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:41:45,422 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:41:56,487 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:41:56,488 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:41:56,488 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:41:56,489 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:41:56,496 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:41:57,497 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:41:58,614 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:42:00,424 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:42:00,424 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:42:15,426 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:42:15,426 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:42:18,748 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:42:18,749 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:42:18,749 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:42:18,751 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:42:19,502 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:42:19,502 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:42:28,615 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:42:30,428 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:42:30,428 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:42:41,263 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:42:41,264 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:42:41,264 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:42:41,265 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:42:41,508 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:42:41,508 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:42:45,430 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:42:45,430 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:42:58,617 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:43:00,432 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:43:00,432 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:43:05,514 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:43:15,434 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:43:15,434 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:43:16,786 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:43:16,787 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:43:16,787 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:43:16,788 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:43:17,517 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:43:17,517 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:43:28,619 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:43:29,157 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:43:29,157 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:43:29,157 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:43:29,158 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:43:29,520 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:43:29,520 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:43:30,436 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:43:30,436 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:43:45,438 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:43:45,438 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:43:50,714 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:43:50,714 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:43:50,715 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:43:50,715 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:43:51,525 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:43:51,525 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:43:58,620 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:44:00,440 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:44:00,440 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:44:01,528 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:44:15,442 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:44:15,442 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:44:22,990 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:44:22,990 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:44:22,991 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:44:22,991 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:44:23,533 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:44:23,534 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:44:28,622 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:44:30,444 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:44:30,444 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:44:44,537 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:44:44,537 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:44:44,537 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:44:44,538 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:44:44,539 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:44:45,446 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:44:45,446 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:44:45,539 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:44:58,624 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:45:00,448 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:45:00,448 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:45:04,133 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:45:04,134 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:45:04,134 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:45:04,135 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:45:04,544 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:45:05,544 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:45:15,450 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:45:15,450 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:45:27,550 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:45:28,626 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:45:30,452 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:45:30,452 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:45:41,311 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:45:41,311 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:45:41,311 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:45:41,312 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:45:41,553 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:45:41,553 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:45:45,454 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:45:45,454 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:45:58,627 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:46:00,456 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:46:00,457 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:46:04,630 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:46:04,630 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:46:04,630 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:46:04,631 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:46:05,559 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:46:05,560 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:46:15,458 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:46:15,459 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:46:22,358 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:46:22,359 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:46:22,359 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:46:22,359 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:46:22,563 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:46:23,564 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:46:28,629 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:46:30,460 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:46:30,461 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:46:45,462 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:46:45,463 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:46:46,570 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:46:58,631 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:47:00,465 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:47:00,465 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:47:06,963 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:47:06,963 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:47:06,964 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:47:06,964 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:47:07,575 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:47:08,575 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:47:15,467 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:47:15,467 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:47:28,490 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:47:28,490 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:47:28,490 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:47:28,492 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:47:28,580 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:47:28,632 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:47:30,469 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:47:30,469 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:47:30,581 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:47:38,719 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:47:38,719 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:47:38,719 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:47:38,720 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:47:39,583 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:47:40,583 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:47:45,471 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:47:45,471 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:47:56,587 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:47:58,634 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:48:00,473 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:48:00,473 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:48:05,912 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:48:05,913 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:48:05,913 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:48:05,914 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:48:06,590 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:48:08,591 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:48:15,475 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:48:15,475 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:48:27,922 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:48:27,922 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:48:27,922 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:48:27,923 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:48:28,596 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:48:28,636 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:48:30,477 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:48:30,477 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:48:30,596 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:48:40,693 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:48:40,694 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:48:40,694 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:48:40,695 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:48:41,599 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:48:42,599 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:48:45,479 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:48:45,479 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:48:56,603 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:48:58,637 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:49:00,482 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:49:00,482 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:49:15,484 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:49:15,484 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:49:16,618 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:49:16,618 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:49:16,618 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:49:16,619 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:49:17,608 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:49:18,608 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:49:26,019 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:49:26,019 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:49:26,019 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:49:26,020 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:49:26,610 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:49:28,611 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:49:28,639 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:49:30,486 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:49:30,486 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:49:36,716 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:49:36,716 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:49:36,717 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:49:36,717 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:49:37,613 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:49:38,614 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:49:45,488 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:49:45,488 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:49:48,616 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:49:58,403 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:49:58,403 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:49:58,403 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:49:58,404 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:49:58,619 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:49:58,640 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:50:00,490 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:50:00,490 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:50:00,619 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:07,741 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:50:07,741 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:50:07,742 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:50:07,742 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:50:08,621 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:08,622 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:50:10,622 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:15,492 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:50:15,492 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:50:18,415 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:50:18,416 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:50:18,416 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:50:18,417 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:50:18,624 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:50:20,625 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:28,642 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:50:30,529 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:50:30,529 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:50:32,628 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:41,905 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:50:41,906 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:50:41,906 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:50:41,907 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:50:42,630 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:50:44,631 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:50:45,544 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:50:45,545 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:50:58,644 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:51:00,546 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:51:00,547 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:51:03,688 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:51:03,689 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:51:03,689 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:51:03,690 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:51:04,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:51:04,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:51:12,879 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:51:12,880 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:51:12,880 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:51:12,881 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:51:13,638 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:51:14,638 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:51:15,549 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:51:15,549 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:51:24,641 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:51:28,646 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:51:30,551 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:51:30,551 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:51:32,544 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:51:32,544 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:51:32,544 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:51:32,545 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:51:32,643 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:51:34,644 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:51:43,598 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:51:43,598 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:51:43,598 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:51:43,599 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:51:43,646 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:51:44,646 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:51:45,553 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:51:45,553 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:51:58,647 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:51:59,221 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:51:59,221 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:51:59,221 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:51:59,222 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:51:59,650 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:52:00,555 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:52:00,555 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:52:00,651 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:52:10,653 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:52:15,557 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:52:15,557 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:52:25,330 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:52:25,330 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:52:25,330 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:52:25,331 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:52:25,657 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:52:26,657 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:52:28,649 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:52:30,559 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:52:30,559 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:52:42,169 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:52:42,169 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:52:42,169 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:52:42,170 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:52:42,661 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:52:44,662 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:52:45,561 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:52:45,561 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:52:58,651 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:53:00,563 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:53:00,563 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:53:03,858 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:53:03,859 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:53:03,859 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:53:03,860 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:53:04,666 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:53:04,667 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:53:15,565 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:53:15,565 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:53:18,670 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:53:27,261 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:53:27,261 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:53:27,262 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:53:27,262 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:53:27,672 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:53:28,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:53:28,672 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:53:30,567 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:53:30,567 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:53:44,095 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:53:44,095 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:53:44,095 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:53:44,096 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:53:44,676 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:53:45,569 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:53:45,569 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:53:46,677 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:53:58,654 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:54:00,571 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:54:00,571 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:54:05,425 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:54:05,425 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:54:05,426 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:54:05,426 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:54:05,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:54:06,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:54:15,573 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:54:15,573 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:54:28,656 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:54:28,688 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:54:30,575 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:54:30,575 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:54:45,577 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:54:45,577 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:54:49,699 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:54:49,700 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:54:49,700 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:54:49,700 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:54:50,693 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:54:50,693 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:54:58,658 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:55:00,579 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:55:00,579 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:55:04,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:55:04,419 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:55:04,419 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:55:04,420 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:55:04,696 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:55:06,697 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:55:14,801 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:55:14,801 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:55:14,801 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:55:14,802 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:55:15,581 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:55:15,581 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:55:15,699 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:55:16,700 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:55:28,659 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:55:28,703 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:55:30,583 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:55:30,583 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:55:45,621 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:55:45,621 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:55:49,792 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:55:49,793 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:55:49,793 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:55:49,794 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:55:50,708 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:55:50,708 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:55:58,661 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:55:58,748 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:55:58,749 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:55:58,749 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:55:58,749 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:55:59,710 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:56:00,623 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:56:00,623 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:56:00,711 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:56:09,762 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:56:09,763 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:56:09,763 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:56:09,763 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:56:10,714 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:56:10,714 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:56:15,625 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:56:15,626 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:56:20,716 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:56:28,663 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:56:30,627 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:56:30,627 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:56:32,870 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:56:32,870 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:56:32,870 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:56:32,871 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:56:33,719 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:56:34,720 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:56:41,204 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:56:41,204 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:56:41,204 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:56:41,205 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:56:41,722 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:56:42,722 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:56:45,629 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:56:45,630 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:56:58,665 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:57:00,631 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:57:00,632 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:57:04,674 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:57:04,674 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:57:04,674 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:57:04,675 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:57:04,727 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:57:06,728 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:57:15,633 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:57:15,634 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:57:26,733 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:57:28,666 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:57:30,636 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:57:30,636 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:57:39,216 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:57:39,217 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:57:39,217 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:57:39,219 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:57:39,736 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:57:40,736 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:57:45,638 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:57:45,638 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:57:54,224 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:57:54,224 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:57:54,224 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:57:54,225 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:57:54,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:57:54,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:57:56,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:57:58,668 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:58:00,639 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:58:00,640 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:58:04,169 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:58:04,169 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:58:04,169 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:58:04,170 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:58:04,742 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:58:04,742 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:58:15,642 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:58:15,642 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:58:28,670 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:58:28,748 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:58:30,644 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:58:30,644 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:58:45,646 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:58:45,647 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:58:48,535 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:58:48,536 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:58:48,536 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:58:48,536 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:58:48,753 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:58:50,754 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:58:58,661 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:58:58,662 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:58:58,662 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:58:58,663 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:58:58,671 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:58:58,756 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:59:00,648 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:59:00,648 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:59:00,756 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:59:08,002 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:59:08,002 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:59:08,003 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:59:08,003 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:59:08,758 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:59:08,759 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:59:15,650 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:59:15,650 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:59:20,762 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:59:28,673 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 18:59:28,711 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:59:28,711 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:59:28,712 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:59:28,712 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:59:28,764 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:59:30,652 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:59:30,652 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:59:30,764 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:59:39,875 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 18:59:39,876 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 18:59:39,876 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 18:59:39,877 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 18:59:40,767 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 18:59:40,767 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 18:59:45,654 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 18:59:45,654 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 18:59:58,675 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:00:00,656 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:00:00,656 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:00:01,857 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:00:01,858 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:00:01,858 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:00:01,859 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:00:02,772 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:00:02,772 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:00:14,775 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:00:15,658 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:00:15,659 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:00:28,676 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:00:30,660 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:00:30,660 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:00:37,818 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:00:37,818 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:00:37,818 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:00:37,819 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:00:38,781 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:00:38,781 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:00:45,662 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:00:45,662 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:00:48,079 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:00:48,079 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:00:48,079 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:00:48,080 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:00:48,784 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:00:48,784 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:00:58,678 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:01:00,331 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:01:00,331 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:01:00,332 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:01:00,333 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:01:00,664 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:01:00,664 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:01:00,788 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:01:00,788 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:01:12,791 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:01:15,666 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:01:15,666 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:01:19,835 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:01:19,835 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:01:19,835 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:01:19,836 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:01:20,794 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:01:20,794 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:01:28,680 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:01:30,668 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:01:30,668 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:01:42,254 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:01:42,254 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:01:42,254 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:01:42,255 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:01:42,799 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:01:42,799 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:01:45,670 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:01:45,671 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:01:53,724 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:01:53,724 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:01:53,725 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:01:53,725 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:01:53,802 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:01:54,802 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:01:58,681 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:02:00,672 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:02:00,672 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:02:04,804 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:02:11,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:02:11,418 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:02:11,418 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:02:11,419 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:02:11,806 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:02:12,806 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:02:15,674 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:02:15,674 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:02:23,793 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:02:23,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:02:23,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:02:23,795 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:02:23,809 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:02:24,810 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:02:28,683 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:02:30,676 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:02:30,676 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:02:34,043 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:02:34,043 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:02:34,043 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:02:34,044 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:02:34,812 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:02:34,812 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:02:45,678 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:02:45,678 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:02:56,817 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:02:58,685 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:03:00,680 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:03:00,680 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:03:06,714 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:03:06,714 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:03:06,714 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:03:06,715 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:03:06,820 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:03:08,821 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:03:15,682 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:03:15,682 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:03:28,687 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:03:29,628 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:03:29,628 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:03:29,628 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:03:29,629 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:03:29,826 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:03:30,684 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:03:30,684 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:03:30,826 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:03:39,193 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:03:39,193 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:03:39,193 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:03:39,194 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:03:39,828 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:03:40,829 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:03:45,686 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:03:45,687 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:03:58,688 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:04:00,688 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:04:00,689 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:04:02,834 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:13,608 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:04:13,608 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:04:13,608 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:04:13,609 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:04:13,837 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:04:14,837 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:15,690 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:04:15,691 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:04:22,256 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:04:22,256 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:04:22,256 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:04:22,257 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:04:22,839 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:22,839 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:04:28,690 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:04:30,692 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:04:30,692 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:04:31,132 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:04:31,132 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:04:31,132 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:04:31,133 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:04:31,842 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:04:32,842 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:44,845 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:45,694 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:04:45,694 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:04:53,168 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:04:53,168 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:04:53,168 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:04:53,169 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:04:53,847 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:04:54,848 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:04:58,692 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:05:00,696 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:05:00,696 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:05:05,209 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:05:05,209 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:05:05,209 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:05:05,210 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:05:05,850 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:05:06,850 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:05:14,544 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:05:14,545 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:05:14,545 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:05:14,546 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:05:14,852 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:05:14,853 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:05:15,698 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:05:15,698 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:05:24,855 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:05:28,693 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:05:30,700 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:05:30,700 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:05:33,555 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:05:33,555 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:05:33,555 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:05:33,556 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:05:33,857 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:05:34,857 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:05:45,702 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:05:45,702 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:05:55,572 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:05:55,573 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:05:55,573 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:05:55,574 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:05:55,863 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:05:56,863 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:05:58,695 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:06:00,704 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:06:00,704 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:06:15,706 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:06:15,706 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:06:19,156 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:06:19,156 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:06:19,156 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:06:19,158 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:06:19,869 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:06:20,869 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:06:28,696 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:06:30,708 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:06:30,708 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:06:34,872 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:06:45,709 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:06:45,710 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:06:57,801 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:06:57,801 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:06:57,801 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:06:57,802 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:06:57,878 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:06:58,698 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:06:58,879 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:07:00,711 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:07:00,711 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:07:15,713 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:07:15,713 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:07:19,733 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:07:19,733 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:07:19,734 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:07:19,734 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:07:19,884 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:07:21,884 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:07:28,700 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:07:30,714 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:07:30,714 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:07:31,241 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:07:31,241 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:07:31,241 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:07:31,242 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:07:31,887 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:07:33,888 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:07:45,716 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:07:45,716 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:07:47,891 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:07:55,928 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:07:55,928 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:07:55,928 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:07:55,929 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:07:56,893 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:07:57,894 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:07:58,701 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:08:00,718 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:08:00,718 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:08:15,719 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:08:15,720 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:08:18,284 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:08:18,285 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:08:18,285 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:08:18,286 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:08:18,899 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:08:19,899 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:08:28,703 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:08:30,721 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:08:30,721 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:08:39,819 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:08:39,819 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:08:39,819 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:08:39,820 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:08:39,904 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:08:41,905 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:08:45,722 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:08:45,723 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:08:51,907 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:08:57,979 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:08:57,979 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:08:57,980 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:08:57,980 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:08:58,705 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:08:58,909 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:08:59,910 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:00,724 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:09:00,724 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:09:10,796 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:09:10,797 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:09:10,797 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:09:10,798 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:09:10,912 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:09:11,913 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:15,726 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:09:15,727 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:09:24,884 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:09:24,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:09:24,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:09:24,886 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:09:24,916 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:09:25,916 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:28,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:09:30,728 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:09:30,728 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:09:35,918 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:43,201 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:09:43,201 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:09:43,201 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:09:43,202 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:09:43,920 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:09:45,730 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:09:45,731 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:09:45,921 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:55,550 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:09:55,551 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:09:55,551 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:09:55,552 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:09:55,924 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:09:57,924 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:09:58,708 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:10:00,732 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:10:00,732 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:10:07,536 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:10:07,536 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:10:07,536 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:10:07,537 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:10:07,927 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:10:09,927 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:10:15,734 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:10:15,734 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:10:17,929 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:10:28,710 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:10:30,736 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:10:30,736 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:10:38,922 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:10:38,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:10:38,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:10:38,924 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:10:38,934 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:10:39,935 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:10:45,738 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:10:45,739 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:10:47,986 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:10:47,986 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:10:47,986 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:10:47,987 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:10:48,937 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:10:49,937 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:10:58,711 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:11:00,198 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:11:00,198 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:11:00,198 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:11:00,199 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:11:00,740 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:11:00,741 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:11:00,940 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:11:01,940 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:11:15,742 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:11:15,743 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:11:23,946 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:11:28,713 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:11:30,744 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:11:30,745 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:11:44,767 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:11:44,767 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:11:44,767 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:11:44,768 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:11:44,951 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:11:45,746 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:11:45,747 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:11:45,951 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:11:58,715 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:12:00,748 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:12:00,748 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:12:08,319 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:12:08,319 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:12:08,320 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:12:08,320 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:12:08,957 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:12:09,957 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:12:15,750 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:12:15,751 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:12:28,716 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:12:30,281 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:12:30,281 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:12:30,282 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:12:30,282 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:12:30,752 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:12:30,752 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:12:30,962 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:12:31,963 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:12:43,965 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:12:45,754 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:12:45,755 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:12:58,718 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:13:00,756 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:13:00,757 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:13:05,259 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:13:05,259 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:13:05,259 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:13:05,260 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:13:05,971 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:13:07,971 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:13:15,758 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:13:15,759 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:13:27,713 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:13:27,713 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:13:27,714 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:13:27,714 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:13:27,976 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:13:28,719 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:13:29,977 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:13:30,760 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:13:30,761 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:13:43,933 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:13:43,933 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:13:43,933 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:13:43,934 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:13:43,980 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:13:45,762 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:13:45,763 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:13:45,981 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:13:58,721 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:14:00,764 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:14:00,764 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:14:09,987 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:15,766 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:14:15,767 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:14:21,288 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:14:21,288 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:14:21,288 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:14:21,289 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:14:21,990 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:21,990 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:14:23,990 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:28,723 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:14:30,387 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:14:30,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:14:30,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:14:30,388 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:14:30,768 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:14:30,768 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:14:30,992 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:14:31,992 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:39,177 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:14:39,177 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:14:39,177 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:14:39,178 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:14:39,995 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:39,995 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:14:45,770 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:14:45,771 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:14:51,998 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:14:58,725 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:15:00,772 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:15:00,773 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:15:01,309 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:15:01,309 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:15:01,309 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:15:01,310 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:15:02,000 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:02,000 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:15:15,774 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:15:15,774 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:15:23,422 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:15:23,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:15:23,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:15:23,423 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:15:24,006 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:15:26,007 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:28,726 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:15:30,776 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:15:30,776 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:15:33,394 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:15:33,394 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:15:33,395 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:15:33,395 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:15:34,009 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:34,009 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:15:36,009 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:44,011 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:45,778 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:15:45,779 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:15:52,875 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:15:52,875 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:15:52,875 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:15:52,876 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:15:53,014 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:15:54,014 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:15:58,729 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:16:00,780 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:16:00,780 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:16:02,814 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:16:02,815 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:16:02,815 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:16:02,816 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:16:03,016 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:16:04,016 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:16:15,782 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:16:15,782 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:16:18,461 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:16:18,462 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:16:18,462 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:16:18,463 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:16:19,020 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:16:20,020 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:16:28,731 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:16:30,784 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:16:30,784 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:16:42,026 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:16:45,786 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:16:45,786 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:16:50,011 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:16:50,011 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:16:50,012 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:16:50,012 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:16:50,027 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:16:52,028 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:16:58,733 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:17:00,788 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:17:00,788 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:17:02,634 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:17:02,634 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:17:02,634 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:17:02,635 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:17:03,031 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:17:04,031 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:17:15,790 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:17:15,790 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:17:24,582 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:17:24,582 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:17:24,582 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:17:24,583 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:17:25,037 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:17:26,038 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:17:28,734 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:17:30,792 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:17:30,792 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:17:36,040 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:17:44,031 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:17:44,031 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:17:44,032 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:17:44,032 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:17:44,042 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:17:45,794 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:17:45,794 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:17:46,042 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:17:54,304 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:17:54,304 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:17:54,304 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:17:54,305 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:17:55,045 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:17:56,045 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:17:58,736 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:18:00,796 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:18:00,796 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:18:06,034 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:18:06,035 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:18:06,035 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:18:06,037 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:18:06,048 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:18:08,048 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:18:15,798 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:18:15,798 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:18:28,737 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:18:30,800 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:18:30,800 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:18:32,054 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:18:45,802 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:18:45,802 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:18:51,377 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:18:51,378 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:18:51,378 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:18:51,379 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:18:52,059 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:18:52,059 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:18:58,739 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:19:00,804 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:19:00,804 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:19:02,412 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:19:02,412 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:19:02,413 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:19:02,413 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:19:03,062 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:19:04,062 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:19:14,587 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:19:14,587 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:19:14,588 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:19:14,588 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:19:15,065 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:19:15,806 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:19:15,806 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:19:16,065 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:19:26,068 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:19:28,741 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:19:30,808 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:19:30,808 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:19:46,160 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:19:46,161 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:19:46,203 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:19:46,203 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:19:46,204 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:19:46,204 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:19:47,074 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:19:48,074 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:19:57,335 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:19:57,336 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:19:57,336 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:19:57,337 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:19:58,076 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:19:58,077 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:19:58,742 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:20:01,163 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:20:01,164 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:20:05,745 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:20:05,745 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:20:05,746 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:20:05,746 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:20:06,078 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:20:08,079 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:20:16,081 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:20:16,166 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:20:16,166 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:20:25,214 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:20:25,214 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:20:25,214 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:20:25,215 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:20:26,084 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:20:26,084 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:20:28,744 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:20:31,168 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:20:31,168 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:20:39,431 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:20:39,431 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:20:39,431 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:20:39,432 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:20:40,087 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:20:40,087 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:20:46,170 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:20:46,170 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:20:58,746 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:21:01,172 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:21:01,172 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:21:01,543 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:21:01,543 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:21:01,544 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:21:01,545 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:21:02,093 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:21:02,093 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:21:16,174 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:21:16,174 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:21:24,098 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:21:28,747 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:21:31,176 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:21:31,176 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:21:46,178 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:21:46,178 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:21:47,102 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:21:47,102 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:21:47,103 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:21:47,103 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:21:47,104 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:21:48,104 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:21:58,749 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:22:01,180 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:22:01,180 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:22:05,455 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:22:05,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:22:05,456 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:22:05,456 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:22:06,109 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:22:06,109 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:22:16,182 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:22:16,182 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:22:27,421 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:22:27,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:22:27,422 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:22:27,422 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:22:28,114 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:22:28,114 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:22:28,751 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:22:31,184 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:22:31,185 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:22:44,118 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:22:46,186 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:22:46,187 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:22:51,025 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:22:51,025 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:22:51,025 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:22:51,026 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:22:51,120 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:22:52,120 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:22:58,752 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:23:01,188 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:23:01,189 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:23:14,378 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:23:14,378 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:23:14,378 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:23:14,380 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:23:15,126 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:23:16,126 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:23:16,190 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:23:16,191 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:23:28,754 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:23:31,192 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:23:31,192 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:23:36,469 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:23:36,469 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:23:36,470 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:23:36,470 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:23:37,131 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:23:38,132 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:23:46,195 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:23:46,195 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:23:48,134 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:23:58,756 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:24:01,199 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:24:01,199 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:24:09,330 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:24:09,331 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:24:09,331 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:24:09,332 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:24:10,139 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:24:10,140 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:24:16,201 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:24:16,201 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:24:18,992 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:24:18,992 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:24:18,992 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:24:18,993 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:24:19,142 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:24:20,142 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:24:28,444 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:24:28,444 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:24:28,445 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:24:28,445 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:24:28,757 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:24:29,144 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:24:30,144 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:24:31,203 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:24:31,203 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:24:42,147 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:24:46,205 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:24:46,205 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:24:55,110 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:24:55,110 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:24:55,111 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:24:55,111 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:24:55,151 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:24:56,151 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:24:58,759 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:25:01,207 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:25:01,207 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:25:05,507 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:25:05,507 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:25:05,507 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:25:05,508 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:25:06,153 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:25:06,154 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:25:16,209 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:25:16,209 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:25:27,434 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:25:27,434 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:25:27,434 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:25:27,435 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:25:28,160 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:25:28,160 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:25:28,760 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:25:31,211 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:25:31,211 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:25:42,163 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:25:46,213 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:25:46,214 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:25:53,793 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:25:53,793 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:25:53,793 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:25:53,794 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:25:54,166 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:25:54,167 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:25:58,762 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:26:01,216 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:26:01,216 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:26:16,218 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:26:16,218 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:26:16,866 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:26:16,866 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:26:16,867 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:26:16,868 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:26:17,172 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:26:18,172 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:26:26,198 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:26:26,198 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:26:26,199 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:26:26,199 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:26:27,175 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:26:28,175 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:26:28,764 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:26:31,220 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:26:31,220 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:26:46,222 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:26:46,222 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:26:50,180 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:26:58,765 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:27:00,904 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:27:00,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:27:00,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:27:00,905 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:27:01,183 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:27:01,224 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:27:01,224 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:27:02,184 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:27:11,232 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:27:11,232 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:27:11,232 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:27:11,233 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:27:12,186 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:27:12,186 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:27:16,226 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:27:16,226 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:27:23,458 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:27:23,458 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:27:23,458 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:27:23,459 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:27:24,189 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:27:24,190 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:27:28,767 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:27:31,228 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:27:31,228 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:27:46,195 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:27:46,230 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:27:46,230 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:27:58,769 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:28:01,232 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:28:01,232 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:28:02,338 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:28:02,338 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:28:02,338 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:28:02,339 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:28:03,199 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:28:05,200 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:28:16,234 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:28:16,234 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:28:25,333 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:28:25,334 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:28:25,334 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:28:25,335 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:28:26,205 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:28:27,205 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:28:28,771 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:28:31,236 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:28:31,236 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:28:37,877 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:28:37,877 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:28:37,877 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:28:37,878 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:28:38,208 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:28:39,208 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:28:46,238 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:28:46,238 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:28:49,211 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:28:56,472 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:28:56,473 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:28:56,473 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:28:56,474 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:28:57,213 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:28:58,772 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:28:59,214 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:29:01,239 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:29:01,240 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:29:06,923 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:29:06,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:29:06,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:29:06,924 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:29:07,216 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:29:09,217 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:29:16,241 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:29:16,242 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:29:23,357 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:29:23,357 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:29:23,357 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:29:23,358 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:29:24,220 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:29:25,221 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:29:28,774 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:29:31,243 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:29:31,244 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:29:46,246 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:29:46,246 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:29:49,227 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:29:58,775 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:29:59,417 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:29:59,417 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:29:59,418 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:29:59,418 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:30:00,230 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:30:01,231 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:30:01,247 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:30:01,248 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:30:08,148 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:30:08,148 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:30:08,148 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:30:08,149 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:30:08,232 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:30:09,233 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:30:16,249 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:30:16,250 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:30:28,777 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:30:31,251 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:30:31,252 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:30:31,942 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:30:31,942 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:30:31,943 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:30:31,943 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:30:32,239 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:30:33,239 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:30:46,253 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:30:46,253 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:30:47,242 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:30:55,201 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:30:55,201 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:30:55,202 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:30:55,202 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:30:55,245 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:30:57,245 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:30:58,779 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:31:01,255 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:31:01,255 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:31:03,998 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:31:03,998 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:31:03,998 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:31:03,999 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:31:04,247 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:31:05,247 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:14,006 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:31:14,007 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:31:14,007 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:31:14,008 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:31:14,250 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:31:15,250 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:16,257 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:31:16,258 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:31:25,252 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:28,781 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:31:31,259 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:31:31,259 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:31:33,496 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:31:33,496 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:31:33,497 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:31:33,497 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:31:34,255 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:31:35,255 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:46,261 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:31:46,261 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:31:47,348 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:31:47,349 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:31:47,349 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:31:47,350 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:31:48,258 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:31:49,259 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:56,423 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:31:56,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:31:56,424 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:31:56,424 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:31:57,261 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:31:57,261 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:31:58,782 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:31:59,261 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:32:01,263 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:32:01,263 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:32:09,264 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:32:16,265 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:32:16,265 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:32:20,498 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:32:20,498 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:32:20,498 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:32:20,499 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:32:21,268 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:32:23,268 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:32:28,784 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:32:31,267 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:32:31,268 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:32:33,610 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:32:33,610 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:32:33,610 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:32:33,611 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:32:34,271 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:32:35,272 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:32:44,512 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:32:44,513 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:32:44,513 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:32:44,514 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:32:45,274 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:32:46,269 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:32:46,269 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:32:47,274 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:32:58,786 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:33:01,271 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:33:01,271 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:33:09,280 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:33:16,273 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:33:16,274 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:33:17,859 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:33:17,859 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:33:17,859 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:33:17,860 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:33:18,282 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:33:19,283 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:33:28,787 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:33:31,275 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:33:31,275 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:33:33,326 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:33:33,326 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:33:33,326 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:33:33,327 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:33:34,287 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:33:35,287 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:33:45,611 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:33:45,611 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:33:45,611 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:33:45,612 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:33:46,277 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:33:46,277 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:33:46,290 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:33:47,290 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:33:57,293 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:33:58,789 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:34:01,279 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:34:01,280 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:34:16,281 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:34:16,281 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:34:18,000 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:34:18,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:34:18,001 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:34:18,002 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:34:18,298 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:34:19,298 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:34:27,660 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:34:27,660 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:34:27,660 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:34:27,661 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:34:28,300 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:34:28,791 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:34:29,301 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:34:31,283 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:34:31,283 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:34:46,285 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:34:46,286 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:34:49,877 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:34:49,877 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:34:49,878 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:34:49,878 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:34:50,306 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:34:51,306 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:34:58,792 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:35:01,288 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:35:01,288 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:35:01,309 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:35:08,652 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:35:08,653 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:35:08,653 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:35:08,654 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:35:09,311 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:35:11,312 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:35:16,290 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:35:16,290 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:35:26,271 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:35:26,271 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:35:26,272 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:35:26,272 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:35:26,315 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:35:27,316 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:35:28,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:35:31,292 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:35:31,292 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:35:46,294 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:35:46,294 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:35:48,261 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:35:48,261 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:35:48,262 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:35:48,262 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:35:48,321 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:35:49,321 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:35:58,796 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:36:01,296 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:36:01,296 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:36:01,325 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:36:12,571 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:36:12,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:36:12,571 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:36:12,572 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:36:13,328 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:36:13,328 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:36:16,298 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:36:16,298 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:36:23,887 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:36:23,887 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:36:23,887 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:36:23,888 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:36:24,330 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:36:25,331 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:36:28,798 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:36:31,300 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:36:31,300 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:36:32,477 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:36:32,478 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:36:32,478 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:36:32,479 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:36:33,333 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:36:33,333 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:36:46,302 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:36:46,302 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:36:55,338 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:36:58,799 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:37:01,304 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:37:01,304 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:37:16,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:37:16,306 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:37:16,923 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:37:16,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:37:16,923 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:37:16,924 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:37:17,343 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:37:19,344 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:37:28,801 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:37:30,885 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:37:30,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:37:30,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:37:30,886 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:37:31,308 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:37:31,308 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:37:31,347 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:37:33,348 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:37:40,439 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:37:40,439 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:37:40,439 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:37:40,440 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:37:41,350 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:37:41,350 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:37:46,310 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:37:46,310 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:37:55,354 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:37:58,803 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:38:01,312 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:38:01,312 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:38:15,862 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:38:15,862 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:38:15,863 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:38:15,863 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:38:16,313 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:38:16,314 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:38:16,359 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:38:17,359 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:38:25,819 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:38:25,819 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:38:25,819 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:38:25,820 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:38:26,362 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:38:27,362 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:38:28,805 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:38:31,315 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:38:31,316 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:38:46,317 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:38:46,318 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:38:48,318 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:38:48,318 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:38:48,319 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:38:48,320 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:38:48,367 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:38:49,368 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:38:58,806 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:39:01,319 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:39:01,320 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:39:09,373 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:39:16,321 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:39:16,322 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:39:28,808 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:39:29,770 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:39:29,770 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:39:29,770 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:39:29,771 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:39:30,378 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:39:31,324 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:39:31,324 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:39:31,378 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:39:46,326 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:39:46,326 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:39:52,362 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:39:52,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:39:52,363 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:39:52,363 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:39:52,383 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:39:53,384 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:39:58,810 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:40:01,026 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:40:01,026 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:40:01,027 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:40:01,027 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:40:01,328 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:40:01,328 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:40:01,386 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:40:03,386 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:40:11,389 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:40:16,330 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:40:16,330 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:40:28,811 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:40:31,332 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:40:31,333 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:40:32,638 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:40:32,638 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:40:32,639 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:40:32,639 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:40:33,394 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:40:33,394 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:40:46,258 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:40:46,258 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:40:46,258 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:40:46,259 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:40:46,335 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:40:46,335 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:40:46,397 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:40:47,398 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:40:58,813 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:41:01,337 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:41:01,337 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:41:01,978 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:41:01,978 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:41:01,979 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:41:01,979 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:41:02,401 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:41:03,401 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:41:16,339 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:41:16,339 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:41:25,407 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:41:28,815 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:41:31,341 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:41:31,342 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:41:32,732 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:41:32,732 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:41:32,733 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:41:32,733 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:41:33,409 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:41:33,410 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:41:46,344 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:41:46,344 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:41:54,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:41:54,306 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:41:54,306 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:41:54,307 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:41:54,415 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:41:55,415 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:41:58,816 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:42:01,346 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:42:01,346 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:42:16,200 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:42:16,200 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:42:16,201 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:42:16,201 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:42:16,347 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:42:16,348 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:42:16,422 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:42:17,422 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:42:28,818 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:42:31,349 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:42:31,349 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:42:31,426 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:42:46,351 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:42:46,351 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:42:51,635 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:42:51,635 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:42:51,635 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:42:51,636 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:42:52,431 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:42:53,432 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:42:58,820 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:43:01,353 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:43:01,353 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:43:14,911 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:43:14,912 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:43:14,912 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:43:14,913 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:43:15,437 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:43:15,437 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:43:16,355 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:43:16,355 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:43:25,265 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:43:25,265 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:43:25,266 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:43:25,266 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:43:25,439 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:43:27,440 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:43:28,821 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:43:31,357 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:43:31,357 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:43:37,442 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:43:46,359 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:43:46,359 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:43:58,823 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:43:59,361 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:43:59,361 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:43:59,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:43:59,362 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:43:59,449 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:44:01,362 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:44:01,362 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:44:01,449 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:44:10,426 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:44:10,426 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:44:10,426 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:44:10,427 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:44:10,452 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:44:11,452 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:44:16,364 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:44:16,364 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:44:28,272 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:44:28,273 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:44:28,273 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:44:28,273 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:44:28,456 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:44:28,825 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:44:29,457 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:44:31,366 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:44:31,366 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:44:39,459 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:44:46,368 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:44:46,369 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:44:58,827 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:45:00,692 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:45:00,692 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:45:00,693 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:45:00,693 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:45:01,371 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:45:01,371 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:45:01,465 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:45:01,466 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:45:16,373 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:45:16,374 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:45:23,752 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:45:23,752 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:45:23,752 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:45:23,753 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:45:24,471 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:45:25,472 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:45:28,828 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:45:31,376 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:45:31,376 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:45:46,378 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:45:46,379 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:45:46,693 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:45:46,694 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:45:46,694 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:45:46,695 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:45:47,477 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:45:47,478 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:45:57,480 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:45:58,830 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:46:01,381 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:46:01,381 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:46:07,214 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:46:07,214 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:46:07,215 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:46:07,216 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:46:07,482 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:46:07,483 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:46:16,383 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:46:16,383 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:46:18,904 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:46:18,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:46:18,905 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:46:18,905 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:46:19,485 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:46:19,486 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:46:28,832 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:46:30,705 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:46:30,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:46:30,706 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:46:30,706 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:46:31,385 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:46:31,386 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:46:31,488 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:46:31,489 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:46:41,491 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:46:46,388 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:46:46,389 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:46:50,373 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:46:50,373 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:46:50,374 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:46:50,374 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:46:50,493 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:46:51,493 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:46:58,833 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:47:01,391 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:47:01,392 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:47:03,349 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:47:03,349 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:47:03,350 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:47:03,350 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:47:03,496 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:47:03,496 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:47:16,394 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:47:16,394 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:47:25,602 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:47:25,602 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:47:25,603 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:47:25,603 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:47:26,502 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:47:27,502 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:47:28,835 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:47:31,396 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:47:31,397 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:47:37,504 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:47:46,399 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:47:46,399 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:47:57,682 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:47:57,682 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:47:57,682 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:47:57,683 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:47:58,510 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:47:58,836 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:47:59,510 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:48:01,401 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:48:01,402 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:48:07,701 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:48:07,701 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:48:07,702 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:48:07,702 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:48:08,512 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:48:09,512 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:48:16,404 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:48:16,404 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:48:28,838 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:48:29,307 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:48:29,307 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:48:29,308 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:48:29,308 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:48:29,517 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:48:29,517 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:48:31,406 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:48:31,406 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:48:43,521 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:48:46,408 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:48:46,409 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:48:58,840 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:49:01,411 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:49:01,411 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:49:04,904 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:49:04,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:49:04,905 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:49:04,906 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:49:05,527 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:49:06,527 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:49:16,413 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:49:16,413 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:49:18,039 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:49:18,039 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:49:18,039 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:49:18,040 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:49:18,530 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:49:20,531 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:49:28,002 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:49:28,002 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:49:28,003 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:49:28,003 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:49:28,533 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:49:28,841 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:49:30,533 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:49:31,415 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:49:31,416 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:49:42,536 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:49:46,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:49:46,418 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:49:52,219 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:49:52,220 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:49:52,220 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:49:52,220 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:49:52,539 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:49:54,540 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:49:58,843 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:50:01,420 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:50:01,420 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:50:15,165 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:50:15,166 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:50:15,166 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:50:15,167 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:50:15,545 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:50:16,422 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:50:16,422 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:50:16,545 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:50:28,845 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:50:31,424 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:50:31,424 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:50:37,614 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:50:37,614 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:50:37,615 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:50:37,615 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:50:38,551 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:50:38,551 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:50:40,551 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:50:46,426 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:50:46,427 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:50:58,846 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:51:01,428 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:51:01,429 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:51:02,557 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:51:11,970 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:51:11,970 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:51:11,970 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:51:11,971 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:51:12,559 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:51:14,560 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:51:16,430 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:51:16,431 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:51:25,215 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:51:25,215 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:51:25,215 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:51:25,216 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:51:25,563 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:51:26,563 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:51:28,848 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:51:31,432 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:51:31,433 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:51:34,011 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:51:34,011 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:51:34,012 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:51:34,012 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:51:34,565 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:51:36,566 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:51:46,435 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:51:46,435 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:51:48,569 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:51:58,850 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:52:01,437 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:52:01,438 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:52:08,836 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:52:08,836 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:52:08,836 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:52:08,837 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:52:09,574 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:52:10,575 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:52:16,439 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:52:16,440 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:52:28,851 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:52:30,651 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:52:30,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:52:30,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:52:30,653 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:52:31,442 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:52:31,442 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:52:31,580 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:52:32,580 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:52:46,444 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:52:46,445 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:52:52,226 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:52:52,226 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:52:52,227 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:52:52,227 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:52:52,585 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:52:54,586 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:52:58,853 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:53:01,446 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:53:01,447 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:53:06,589 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:53:15,986 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:53:15,986 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:53:15,987 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:53:15,988 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:53:16,448 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:53:16,449 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:53:16,591 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:53:18,592 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:53:25,983 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:53:25,983 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:53:25,983 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:53:25,984 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:53:26,594 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:53:28,595 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:53:28,855 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:53:31,450 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:53:31,450 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:53:46,452 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:53:46,452 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:53:47,958 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:53:47,958 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:53:47,958 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:53:47,959 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:53:48,600 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:53:50,600 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:53:58,856 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:54:01,454 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:54:01,455 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:54:06,604 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:54:15,158 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:54:15,159 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:54:15,159 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:54:15,160 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:54:15,607 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:54:16,456 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:54:16,456 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:54:16,607 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:54:28,858 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:54:31,459 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:54:31,459 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:54:38,144 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:54:38,144 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:54:38,145 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:54:38,145 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:54:38,612 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:54:40,613 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:54:46,461 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:54:46,461 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:54:48,761 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:54:48,761 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:54:48,761 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:54:48,762 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:54:49,615 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:54:50,616 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:54:58,860 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:55:00,618 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:55:01,463 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:55:01,463 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:55:12,423 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:55:12,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:55:12,423 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:55:12,424 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:55:12,621 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:55:14,622 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:55:16,465 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:55:16,465 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:55:23,166 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:55:23,166 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:55:23,166 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:55:23,167 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:55:23,624 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:55:24,624 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:55:28,861 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:55:31,467 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:55:31,467 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:55:46,312 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:55:46,312 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:55:46,313 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:55:46,313 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:55:46,469 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:55:46,469 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:55:46,630 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:55:48,631 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:55:58,633 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:55:58,863 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:56:01,470 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:56:01,471 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:56:07,256 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:56:07,256 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:56:07,257 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:56:07,257 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:56:07,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:56:08,636 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:56:16,472 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:56:16,473 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:56:19,693 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:56:19,693 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:56:19,693 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:56:19,694 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:56:20,639 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:56:20,639 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:56:28,865 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:56:29,325 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:56:29,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:56:29,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:56:29,326 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:56:29,642 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:56:30,642 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:56:31,475 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:56:31,475 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:56:42,645 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:56:46,477 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:56:46,477 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:56:51,477 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:56:51,478 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:56:51,478 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:56:51,479 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:56:51,648 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:56:52,648 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:56:58,866 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:57:01,479 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:57:01,479 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:57:02,650 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:57:02,651 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:57:02,652 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:57:02,652 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:57:03,652 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:57:04,652 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:57:13,048 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:57:13,048 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:57:13,048 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:57:13,049 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:57:13,654 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:57:14,655 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:57:16,481 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:57:16,482 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:57:24,657 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:57:28,868 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:57:31,484 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:57:31,484 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:57:45,371 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:57:45,371 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:57:45,372 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:57:45,372 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:57:45,664 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:57:46,486 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:57:46,486 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:57:46,664 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:57:58,869 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:58:01,488 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:58:01,488 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:58:08,325 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:58:08,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:58:08,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:58:08,326 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:58:08,670 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:58:10,670 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:16,490 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:58:16,491 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:58:20,124 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:58:20,124 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:58:20,124 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:58:20,125 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:58:20,673 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:20,673 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:58:22,673 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:28,871 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:58:31,601 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:58:31,601 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:58:32,676 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:42,486 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:58:42,486 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:58:42,487 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:58:42,487 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:58:42,679 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:58:44,679 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:46,611 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:58:46,612 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:58:54,106 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:58:54,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:58:54,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:58:54,107 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:58:54,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:58:54,682 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:58:58,873 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:59:01,613 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:59:01,614 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:59:03,983 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:59:03,984 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:59:03,984 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:59:03,985 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:59:04,684 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:59:04,684 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:59:16,615 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:59:16,616 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:59:26,690 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:59:28,875 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 19:59:31,618 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:59:31,618 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:59:35,558 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:59:35,559 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:59:35,559 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:59:35,560 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:59:35,692 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:59:36,693 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:59:44,600 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 19:59:44,600 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 19:59:44,601 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 19:59:44,601 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 19:59:44,695 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 19:59:46,620 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 19:59:46,620 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 19:59:46,695 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 19:59:58,876 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:00:01,622 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:00:01,622 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:00:07,794 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:00:07,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:00:07,795 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:00:07,795 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:00:08,701 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:00:08,701 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:00:16,624 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:00:16,625 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:00:28,878 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:00:31,626 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:00:31,627 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:00:32,707 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:00:46,629 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:00:46,630 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:00:53,431 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:00:53,431 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:00:53,432 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:00:53,432 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:00:53,712 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:00:54,713 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:00:58,880 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:01:01,633 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:01:01,633 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:01:05,099 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:01:05,099 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:01:05,100 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:01:05,101 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:01:05,715 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:01:06,716 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:01:16,635 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:01:16,636 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:01:22,567 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:01:22,567 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:01:22,567 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:01:22,568 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:01:22,720 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:01:24,720 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:01:28,881 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:01:31,638 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:01:31,638 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:01:46,640 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:01:46,640 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:01:46,726 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:01:58,883 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:02:01,642 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:02:01,642 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:02:05,346 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:02:05,346 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:02:05,347 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:02:05,347 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:02:05,731 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:02:06,731 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:02:15,467 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:02:15,467 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:02:15,468 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:02:15,468 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:02:15,733 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:02:16,644 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:02:16,644 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:02:16,734 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:02:27,118 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:02:27,118 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:02:27,119 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:02:27,119 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:02:27,736 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:02:28,737 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:02:28,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:02:31,646 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:02:31,646 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:02:40,740 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:02:46,648 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:02:46,648 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:02:51,101 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:02:51,101 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:02:51,102 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:02:51,102 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:02:51,743 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:02:52,743 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:02:58,886 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:03:00,585 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:03:00,585 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:03:00,586 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:03:00,586 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:03:00,745 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:03:01,650 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:03:01,650 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:03:02,745 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:03:10,252 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:03:10,252 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:03:10,253 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:03:10,253 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:03:10,748 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:03:10,748 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:03:16,652 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:03:16,652 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:03:22,751 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:03:28,888 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:03:31,654 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:03:31,655 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:03:39,153 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:03:39,153 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:03:39,154 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:03:39,154 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:03:39,755 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:03:40,755 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:03:46,656 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:03:46,657 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:03:58,889 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:04:02,062 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:04:02,062 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:04:02,104 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:04:02,104 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:04:02,104 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:04:02,105 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:04:02,761 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:04:02,761 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:04:13,085 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:04:13,086 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:04:13,086 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:04:13,087 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:04:13,764 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:04:14,764 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:04:17,064 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:04:17,065 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:04:24,767 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:04:28,891 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:04:32,067 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:04:32,067 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:04:35,998 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:04:35,998 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:04:35,999 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:04:36,000 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:04:36,770 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:04:36,770 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:04:47,069 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:04:47,069 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:04:57,002 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:04:57,002 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:04:57,002 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:04:57,003 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:04:57,775 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:04:58,775 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:04:58,893 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:05:02,071 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:05:02,071 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:05:10,366 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:05:10,366 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:05:10,366 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:05:10,367 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:05:10,778 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:05:10,778 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:05:17,074 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:05:17,074 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:05:22,781 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:05:28,894 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:05:32,076 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:05:32,076 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:05:32,520 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:05:32,520 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:05:32,521 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:05:32,521 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:05:32,784 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:05:32,784 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:05:47,078 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:05:47,078 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:05:54,302 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:05:54,302 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:05:54,302 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:05:54,303 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:05:54,789 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:05:54,790 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:05:58,896 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:06:02,080 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:06:02,081 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:06:17,083 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:06:17,083 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:06:17,343 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:06:17,344 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:06:17,345 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:06:17,346 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:06:17,795 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:06:18,796 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:06:26,798 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:06:28,898 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:06:32,085 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:06:32,085 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:06:36,551 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:06:36,552 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:06:36,552 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:06:36,553 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:06:36,800 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:06:36,800 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:06:47,087 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:06:47,088 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:06:49,172 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:06:49,172 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:06:49,172 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:06:49,173 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:06:49,803 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:06:50,804 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:06:58,899 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:06:59,301 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:06:59,301 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:06:59,301 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:06:59,302 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:06:59,806 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:07:00,806 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:07:02,089 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:07:02,090 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:07:17,091 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:07:17,092 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:07:22,812 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:07:28,901 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:07:32,093 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:07:32,094 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:07:33,905 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:07:33,905 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:07:33,906 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:07:33,906 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:07:34,815 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:07:34,815 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:07:46,752 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:07:46,752 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:07:46,753 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:07:46,753 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:07:46,818 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:07:47,095 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:07:47,095 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:07:48,818 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:07:58,903 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:08:02,098 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:08:02,098 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:08:03,666 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:08:03,666 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:08:03,666 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:08:03,667 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:08:03,822 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:08:04,822 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:08:17,100 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:08:17,100 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:08:26,827 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:08:28,904 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:08:32,102 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:08:32,103 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:08:47,104 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:08:47,105 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:08:48,174 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:08:48,174 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:08:48,174 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:08:48,175 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:08:48,832 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:08:48,833 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:08:58,906 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:08:59,370 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:08:59,370 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:08:59,370 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:08:59,371 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:08:59,835 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:09:01,836 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:09:02,106 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:09:02,107 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:09:17,108 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:09:17,109 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:09:22,418 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:09:22,418 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:09:22,418 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:09:22,419 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:09:22,840 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:09:23,841 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:09:28,908 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:09:32,110 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:09:32,111 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:09:33,843 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:09:42,539 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:09:42,539 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:09:42,540 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:09:42,541 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:09:42,845 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:09:43,846 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:09:47,112 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:09:47,113 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:09:55,195 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:09:55,196 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:09:55,196 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:09:55,197 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:09:55,849 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:09:57,849 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:09:58,909 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:10:02,114 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:10:02,115 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:10:10,230 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:10:10,230 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:10:10,231 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:10:10,231 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:10:10,852 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:10:11,853 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:10:17,117 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:10:17,117 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:10:28,911 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:10:29,857 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:10:32,119 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:10:32,119 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:10:36,354 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:10:36,354 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:10:36,354 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:10:36,355 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:10:36,859 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:10:37,859 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:10:47,121 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:10:47,121 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:10:51,702 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:10:51,702 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:10:51,702 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:10:51,703 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:10:51,862 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:10:53,863 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:10:58,912 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:11:00,772 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:11:00,772 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:11:00,773 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:11:00,774 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:11:00,865 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:11:01,865 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:11:02,123 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:11:02,123 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:11:15,868 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:11:17,125 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:11:17,125 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:11:28,914 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:11:32,127 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:11:32,127 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:11:35,718 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:11:35,718 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:11:35,718 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:11:35,719 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:11:35,873 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:11:37,873 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:11:47,129 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:11:47,129 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:11:58,106 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:11:58,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:11:58,106 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:11:58,107 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:11:58,878 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:11:58,916 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:11:59,879 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:12:02,131 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:12:02,131 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:12:09,288 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:12:09,288 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:12:09,289 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:12:09,289 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:12:09,881 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:12:11,882 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:12:17,133 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:12:17,133 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:12:21,884 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:12:28,917 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:12:30,899 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:12:30,899 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:12:30,899 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:12:30,900 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:12:31,887 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:12:31,887 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:12:32,135 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:12:32,136 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:12:47,138 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:12:47,138 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:12:52,703 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:12:52,703 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:12:52,703 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:12:52,704 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:12:52,892 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:12:53,892 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:12:58,919 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:13:02,221 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:13:02,222 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:13:02,223 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:13:02,223 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:13:02,224 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:13:02,224 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:13:02,895 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:13:03,895 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:13:17,223 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:13:17,224 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:13:25,900 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:13:28,921 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:13:32,226 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:13:32,226 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:13:36,138 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:13:36,139 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:13:36,139 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:13:36,140 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:13:36,903 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:13:37,903 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:13:45,554 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:13:45,554 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:13:45,554 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:13:45,555 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:13:45,905 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:13:47,227 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:13:47,228 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:13:47,906 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:13:58,915 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:13:58,915 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:13:58,916 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:13:58,916 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:13:58,922 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:13:59,909 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:13:59,909 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:14:02,229 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:14:02,230 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:14:17,231 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:14:17,232 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:14:23,915 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:14:28,924 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:14:32,234 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:14:32,234 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:14:32,734 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:14:32,734 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:14:32,735 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:14:32,735 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:14:32,917 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:14:33,917 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:14:47,236 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:14:47,236 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:14:54,663 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:14:54,664 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:14:54,664 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:14:54,665 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:14:54,922 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:14:55,923 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:14:58,925 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:15:02,238 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:15:02,239 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:15:10,794 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:15:10,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:15:10,794 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:15:10,795 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:15:10,928 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:15:11,928 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:15:17,240 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:15:17,241 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:15:28,927 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:15:31,934 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:15:32,243 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:15:32,243 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:15:40,768 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:15:40,768 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:15:40,768 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:15:40,769 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:15:40,936 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:15:41,936 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:15:47,246 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:15:47,246 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:15:58,929 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:16:02,248 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:16:02,248 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:16:03,834 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:16:03,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:16:03,834 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:16:03,835 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:16:03,941 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:16:05,943 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:16:17,250 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:16:17,250 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:16:24,970 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:16:24,971 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:16:24,971 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:16:24,972 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:16:25,947 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:16:25,947 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:16:28,931 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:16:32,252 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:16:32,252 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:16:47,254 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:16:47,254 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:16:47,953 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:16:58,932 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:17:02,256 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:17:02,256 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:17:09,638 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:17:09,638 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:17:09,638 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:17:09,639 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:17:09,959 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:17:11,959 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:17:17,258 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:17:17,258 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:17:23,870 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:17:23,870 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:17:23,871 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:17:23,871 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:17:23,962 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:17:25,963 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:17:28,934 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:17:32,260 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:17:32,260 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:17:46,325 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:17:46,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:17:46,325 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:17:46,327 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:17:46,968 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:17:47,262 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:17:47,262 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:17:47,968 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:17:58,936 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:18:02,264 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:18:02,264 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:18:05,973 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:18:11,736 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:18:11,736 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:18:11,737 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:18:11,737 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:18:11,974 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:18:13,975 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:18:17,266 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:18:17,266 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:18:21,840 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:18:21,840 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:18:21,840 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:18:21,841 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:18:21,977 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:18:23,977 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:18:28,937 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:18:32,268 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:18:32,268 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:18:44,888 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:18:44,889 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:18:44,889 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:18:44,889 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:18:44,982 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:18:45,983 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:18:47,270 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:18:47,270 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:18:55,986 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:18:58,940 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:19:02,272 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:19:02,272 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:19:14,387 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:19:14,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:19:14,387 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:19:14,388 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:19:14,992 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:19:15,992 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:19:17,274 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:19:17,274 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:19:28,091 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:19:28,091 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:19:28,091 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:19:28,092 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:19:28,941 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:19:28,995 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:19:29,996 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:19:32,276 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:19:32,277 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:19:37,737 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:19:37,737 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:19:37,737 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:19:37,739 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:19:37,998 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:19:39,998 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:19:47,279 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:19:47,279 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:19:50,001 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:19:58,943 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:20:00,455 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:20:00,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:20:00,455 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:20:00,456 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:20:01,004 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:20:02,004 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:20:02,281 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:20:02,282 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:20:12,827 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:20:12,827 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:20:12,827 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:20:12,828 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:20:13,007 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:20:14,008 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:20:17,284 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:20:17,284 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:20:28,944 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:20:32,286 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:20:32,286 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:20:35,296 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:20:35,297 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:20:35,297 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:20:35,298 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:20:36,013 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:20:36,013 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:20:47,288 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:20:47,289 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:20:48,016 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:20:58,078 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:20:58,078 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:20:58,078 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:20:58,079 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:20:58,946 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:20:59,019 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:21:00,019 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:21:02,291 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:21:02,291 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:21:07,805 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:21:07,805 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:21:07,805 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:21:07,806 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:21:08,021 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:21:10,022 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:21:17,293 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:21:17,293 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:21:23,950 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:21:23,950 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:21:23,950 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:21:23,951 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:21:24,025 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:21:26,026 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:21:28,948 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:21:32,295 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:21:32,296 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:21:36,028 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:21:46,627 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:21:46,628 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:21:46,628 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:21:46,628 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:21:47,033 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:21:47,298 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:21:47,298 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:21:48,033 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:21:58,950 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:22:01,318 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:22:01,318 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:22:01,319 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:22:01,319 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:22:02,036 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:22:02,037 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:22:02,300 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:22:02,300 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:22:12,972 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:22:12,972 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:22:12,972 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:22:12,973 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:22:13,039 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:22:14,040 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:22:17,302 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:22:17,302 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:22:24,042 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:22:28,952 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:22:31,251 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:22:31,251 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:22:31,251 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:22:31,252 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:22:32,044 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:22:32,044 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:22:32,304 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:22:32,304 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:22:43,890 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:22:43,890 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:22:43,891 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:22:43,892 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:22:44,047 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:22:46,048 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:22:47,306 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:22:47,307 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:22:58,953 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:23:02,309 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:23:02,309 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:23:06,307 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:23:06,307 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:23:06,308 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:23:06,308 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:23:07,053 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:23:08,053 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:23:17,311 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:23:17,311 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:23:20,056 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:23:26,337 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:23:26,337 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:23:26,338 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:23:26,338 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:23:27,058 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:23:28,058 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:23:28,955 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:23:32,313 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:23:32,314 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:23:47,316 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:23:47,316 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:23:49,243 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:23:49,244 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:23:49,244 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:23:49,245 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:23:50,064 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:23:50,064 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:23:58,957 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:24:02,318 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:24:02,318 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:24:05,362 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:24:05,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:24:05,362 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:24:05,363 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:24:06,068 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:24:06,068 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:24:17,320 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:24:17,321 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:24:28,074 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:24:28,958 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:24:32,323 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:24:32,323 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:24:36,872 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:24:36,872 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:24:36,872 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:24:36,873 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:24:37,076 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:24:38,077 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:24:47,325 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:24:47,325 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:24:58,960 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:24:59,439 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:24:59,439 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:24:59,440 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:24:59,440 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:25:00,083 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:25:00,083 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:25:02,327 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:25:02,328 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:25:17,329 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:25:17,330 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:25:23,125 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:25:23,125 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:25:23,125 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:25:23,126 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:25:24,089 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:25:24,090 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:25:28,962 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:25:32,332 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:25:32,332 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:25:34,092 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:25:47,334 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:25:47,335 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:25:49,272 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:25:49,272 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:25:49,272 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:25:49,273 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:25:50,096 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:25:50,097 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:25:58,963 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:25:59,777 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:25:59,777 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:25:59,777 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:25:59,778 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:26:00,099 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:26:00,099 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:26:02,336 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:26:02,337 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:26:08,850 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:26:08,851 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:26:08,851 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:26:08,851 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:26:09,102 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:26:10,102 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:26:17,339 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:26:17,339 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:26:28,965 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:26:32,108 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:26:32,341 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:26:32,341 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:26:47,343 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:26:47,344 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:26:50,885 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:26:50,885 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:26:50,886 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:26:50,886 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:26:51,113 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:26:52,114 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:26:58,967 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:27:02,346 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:27:02,346 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:27:13,371 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:27:13,371 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:27:13,372 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:27:13,372 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:27:14,119 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:27:14,120 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:27:17,348 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:27:17,348 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:27:24,889 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: partial_history +2024-05-07 20:27:24,889 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:27:24,889 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:27:24,890 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:27:25,122 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:27:26,123 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:27:28,969 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:27:32,350 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:27:32,351 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:27:36,126 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:27:47,402 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:27:47,403 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:27:58,970 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:28:03,142 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: stop_status +2024-05-07 20:28:03,142 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: stop_status +2024-05-07 20:28:03,758 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: telemetry +2024-05-07 20:28:03,758 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: exit +2024-05-07 20:28:03,758 INFO SenderThread:125735 [tracking_sender.py:send_exit():438] handling exit code: 0 +2024-05-07 20:28:03,759 INFO SenderThread:125735 [tracking_sender.py:send_exit():440] handling runtime: 10925 +2024-05-07 20:28:03,759 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:28:03,759 INFO SenderThread:125735 [tracking_sender.py:send_exit():446] send defer +2024-05-07 20:28:03,759 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,760 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 0 +2024-05-07 20:28:03,760 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,760 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 0 +2024-05-07 20:28:03,760 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 1 +2024-05-07 20:28:03,760 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,760 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 1 +2024-05-07 20:28:03,760 INFO HandlerThread:125735 [system_monitor.py:finish():160] Stopping system monitor +2024-05-07 20:28:03,765 INFO HandlerThread:125735 [interfaces.py:finish():173] Joined cpu +2024-05-07 20:28:03,765 DEBUG SystemMonitor:125735 [system_monitor.py:_start():137] Finished system metrics aggregation loop +2024-05-07 20:28:03,766 DEBUG SystemMonitor:125735 [system_monitor.py:_start():141] Publishing last batch of metrics +2024-05-07 20:28:03,767 INFO HandlerThread:125735 [interfaces.py:finish():173] Joined disk +2024-05-07 20:28:03,802 INFO HandlerThread:125735 [interfaces.py:finish():173] Joined gpu +2024-05-07 20:28:03,802 INFO HandlerThread:125735 [interfaces.py:finish():173] Joined memory +2024-05-07 20:28:03,802 INFO HandlerThread:125735 [interfaces.py:finish():173] Joined network +2024-05-07 20:28:03,802 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,802 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 1 +2024-05-07 20:28:03,803 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 2 +2024-05-07 20:28:03,803 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: stats +2024-05-07 20:28:03,804 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,804 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 2 +2024-05-07 20:28:03,804 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: history +2024-05-07 20:28:03,804 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:28:03,805 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:28:03,805 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,805 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 2 +2024-05-07 20:28:03,805 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 3 +2024-05-07 20:28:03,805 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,805 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 3 +2024-05-07 20:28:03,805 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,805 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 3 +2024-05-07 20:28:03,805 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 4 +2024-05-07 20:28:03,806 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,806 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 4 +2024-05-07 20:28:03,806 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: summary +2024-05-07 20:28:03,806 INFO SenderThread:125735 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end +2024-05-07 20:28:03,806 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,806 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 4 +2024-05-07 20:28:03,806 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 5 +2024-05-07 20:28:03,806 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,806 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 5 +2024-05-07 20:28:03,807 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,807 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 5 +2024-05-07 20:28:03,807 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 6 +2024-05-07 20:28:03,807 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:03,807 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 6 +2024-05-07 20:28:03,807 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:03,807 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 6 +2024-05-07 20:28:04,133 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/config.yaml +2024-05-07 20:28:04,133 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:28:04,759 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: poll_exit +2024-05-07 20:28:05,256 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 7 +2024-05-07 20:28:05,256 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: poll_exit +2024-05-07 20:28:05,257 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:05,257 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 7 +2024-05-07 20:28:05,257 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:05,257 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 7 +2024-05-07 20:28:05,257 INFO SenderThread:125735 [dir_watcher.py:finish():362] shutting down directory watcher +2024-05-07 20:28:05,759 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: poll_exit +2024-05-07 20:28:06,134 INFO Thread-15 :125735 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:28:06,134 INFO SenderThread:125735 [dir_watcher.py:finish():392] scan: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files +2024-05-07 20:28:06,135 INFO SenderThread:125735 [dir_watcher.py:finish():406] scan save: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log output.log +2024-05-07 20:28:06,135 INFO SenderThread:125735 [dir_watcher.py:finish():406] scan save: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/requirements.txt requirements.txt +2024-05-07 20:28:06,135 INFO SenderThread:125735 [dir_watcher.py:finish():406] scan save: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json wandb-summary.json +2024-05-07 20:28:06,135 INFO SenderThread:125735 [dir_watcher.py:finish():406] scan save: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/config.yaml config.yaml +2024-05-07 20:28:06,135 INFO SenderThread:125735 [dir_watcher.py:finish():406] scan save: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-metadata.json wandb-metadata.json +2024-05-07 20:28:06,135 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 8 +2024-05-07 20:28:06,135 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: poll_exit +2024-05-07 20:28:06,137 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:06,138 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 8 +2024-05-07 20:28:06,139 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:06,140 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 8 +2024-05-07 20:28:06,141 INFO SenderThread:125735 [file_pusher.py:finish():168] shutting down file pusher +2024-05-07 20:28:06,185 INFO Thread-22 :125735 [upload_job.py:push():143] Uploaded file None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/config.yaml +2024-05-07 20:28:06,188 INFO Thread-20 :125735 [upload_job.py:push():143] Uploaded file None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/requirements.txt +2024-05-07 20:28:06,209 INFO Thread-21 :125735 [upload_job.py:push():143] Uploaded file None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/wandb-summary.json +2024-05-07 20:28:06,210 INFO Thread-19 :125735 [upload_job.py:push():143] Uploaded file None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/files/output.log +2024-05-07 20:28:06,410 INFO Thread-14 :125735 [tracking_sender.py:transition_state():459] send defer: 9 +2024-05-07 20:28:06,411 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:06,411 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 9 +2024-05-07 20:28:06,411 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:06,411 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 9 +2024-05-07 20:28:06,411 INFO SenderThread:125735 [file_pusher.py:join():173] waiting for file pusher +2024-05-07 20:28:06,411 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 10 +2024-05-07 20:28:06,411 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:06,411 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 10 +2024-05-07 20:28:06,411 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:06,411 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 10 +2024-05-07 20:28:06,565 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 11 +2024-05-07 20:28:06,565 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:06,565 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 11 +2024-05-07 20:28:06,565 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:06,565 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 11 +2024-05-07 20:28:06,566 INFO SenderThread:125735 [tracking_sender.py:transition_state():459] send defer: 12 +2024-05-07 20:28:06,566 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: final +2024-05-07 20:28:06,566 DEBUG SenderThread:125735 [tracking_sender.py:send():315] send: footer +2024-05-07 20:28:06,566 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: defer +2024-05-07 20:28:06,566 INFO HandlerThread:125735 [handler.py:handle_request_defer():163] handle defer: 12 +2024-05-07 20:28:06,566 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: defer +2024-05-07 20:28:06,566 INFO SenderThread:125735 [tracking_sender.py:send_request_defer():455] handle sender defer: 12 +2024-05-07 20:28:06,568 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: poll_exit +2024-05-07 20:28:06,568 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: server_info +2024-05-07 20:28:06,568 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: poll_exit +2024-05-07 20:28:06,568 DEBUG SenderThread:125735 [tracking_sender.py:send_request():329] send_request: server_info +2024-05-07 20:28:06,569 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: get_summary +2024-05-07 20:28:06,569 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: sampled_history +2024-05-07 20:28:06,610 DEBUG HandlerThread:125735 [handler.py:handle_request():140] handle_request: shutdown +2024-05-07 20:28:06,610 INFO HandlerThread:125735 [handler.py:finish():817] shutting down handler +2024-05-07 20:28:07,566 INFO WriterThread:125735 [datastore.py:close():283] close: None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/run-itlu7dts.wandb +2024-05-07 20:28:07,568 INFO SenderThread:125735 [tracking_sender.py:finish():1338] shutting down sender +2024-05-07 20:28:07,569 INFO SenderThread:125735 [file_pusher.py:finish():168] shutting down file pusher +2024-05-07 20:28:07,569 INFO SenderThread:125735 [file_pusher.py:join():173] waiting for file pusher +2024-05-07 20:28:08,821 INFO MainThread:125735 [internal.py:handle_exit():78] Internal process exited