diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,2836 +1,2002 @@ -2024-03-28 02:58:27,478 INFO StreamThr :118 [internal.py:wandb_internal():86] W&B internal server running at pid: 118, started at: 2024-03-28 02:58:27.477747 -2024-03-28 02:58:27,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status -2024-03-28 02:58:27,828 INFO WriterThread:118 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/run-lnd0ipu3.wandb -2024-03-28 02:58:27,829 DEBUG SenderThread:118 [sender.py:send():378] send: header -2024-03-28 02:58:27,833 DEBUG SenderThread:118 [sender.py:send():378] send: run -2024-03-28 02:58:27,999 INFO SenderThread:118 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files -2024-03-28 02:58:28,000 INFO SenderThread:118 [sender.py:_start_run_threads():1099] run started: lnd0ipu3 with start time 1711594707.478978 -2024-03-28 02:58:28,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: check_version -2024-03-28 02:58:28,001 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: check_version -2024-03-28 02:58:28,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: run_start -2024-03-28 02:58:28,103 DEBUG HandlerThread:118 [system_info.py:__init__():26] System info init -2024-03-28 02:58:28,104 DEBUG HandlerThread:118 [system_info.py:__init__():41] System info init done -2024-03-28 02:58:28,104 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 02:58:28,104 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 02:58:28,104 INFO HandlerThread:118 [system_monitor.py:probe():214] Collecting system info -2024-03-28 02:58:28,105 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 02:58:28,105 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 02:58:28,106 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 02:58:28,107 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 02:58:28,108 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 02:58:28,118 DEBUG HandlerThread:118 [system_info.py:probe():150] Probing system -2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [gitlib.py:_init_repo():56] git repository is invalid -2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_info.py:probe():198] Probing system done -2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-28T02:58:28.119009', 'startedAt': '2024-03-28T02:58:27.470892', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'dbef23be6360', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.196, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5533.654880523682}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} -2024-03-28 02:58:28,121 INFO HandlerThread:118 [system_monitor.py:probe():224] Finished collecting system info -2024-03-28 02:58:28,121 INFO HandlerThread:118 [system_monitor.py:probe():227] Publishing system info -2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-03-28 02:58:29,002 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/conda-environment.yaml -2024-03-28 02:59:25,020 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/conda-environment.yaml -2024-03-28 02:59:25,069 DEBUG HandlerThread:118 [system_info.py:_save_conda():219] Saving conda packages done -2024-03-28 02:59:25,069 INFO HandlerThread:118 [system_monitor.py:probe():229] Finished publishing system info -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: python_packages -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: stop_status -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:25,082 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: python_packages -2024-03-28 02:59:25,085 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry -2024-03-28 02:59:25,085 DEBUG SenderThread:118 [sender.py:send():378] send: config -2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry -2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: stop_status -2024-03-28 02:59:25,123 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 02:59:25,123 WARNING SenderThread:118 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) -2024-03-28 02:59:25,123 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry -2024-03-28 02:59:25,124 DEBUG SenderThread:118 [sender.py:send():378] send: files -2024-03-28 02:59:25,124 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now -2024-03-28 02:59:25,326 INFO wandb-upload_0:118 [upload_job.py:push():131] Uploaded file /tmp/tmpcqrallmywandb/ogr9175t-wandb-metadata.json -2024-03-28 02:59:26,020 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/requirements.txt -2024-03-28 02:59:26,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 02:59:26,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-metadata.json -2024-03-28 02:59:28,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 02:59:28,108 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-28 02:59:28,110 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 02:59:28,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:29,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml -2024-03-28 02:59:29,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:33,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:34,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:38,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:39,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:43,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:44,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:48,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:49,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:53,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:54,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 02:59:58,111 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 02:59:59,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 02:59:59,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:04,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:04,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:09,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:09,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:10,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:00:10,013 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:00:10,036 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:00:14,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:15,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:16,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:00:16,861 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:00:17,039 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:00:19,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:20,040 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:00:20,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:24,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:25,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:28,112 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:00:29,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:31,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:32,045 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml -2024-03-28 03:00:34,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:36,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:39,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:41,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:44,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:46,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:49,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:51,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:54,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:00:56,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:00:58,113 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:00:59,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:02,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:04,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:07,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:09,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:12,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:14,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:17,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:19,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:22,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:24,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:27,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:28,114 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:01:28,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:01:28,956 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:01:28,956 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:01:28,956 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:01:29,068 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:01:29,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:32,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:34,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:35,737 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:01:35,738 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:01:35,739 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:01:35,739 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:01:36,071 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:01:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:38,071 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:01:39,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:43,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:44,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:48,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:49,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:53,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:54,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:01:58,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:01:58,115 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:01:59,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:03,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:04,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:08,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:09,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:13,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:14,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:18,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:19,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:23,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:24,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:28,116 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:02:29,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:29,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:34,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:34,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:39,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:39,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:44,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:44,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:46,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:02:46,820 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:02:46,820 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:02:46,823 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:02:47,100 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:02:49,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:49,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:53,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:02:53,572 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:02:53,572 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:02:53,573 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:02:54,103 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:02:54,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:02:55,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:02:56,104 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:02:58,117 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:02:59,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:01,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:04,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:06,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:09,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:11,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:14,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:16,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:19,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:21,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:24,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:26,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:28,118 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:03:29,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:32,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:34,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:37,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:39,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:42,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:44,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:47,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:49,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:52,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:54,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:03:57,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:03:58,119 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:03:59,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:03,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:04,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:05,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:04:05,942 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:04:05,942 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:04:05,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:04:06,131 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:04:08,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:09,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:12,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:04:12,647 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:04:12,647 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:04:12,648 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:04:13,134 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:04:14,134 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:04:14,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:14,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:19,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:19,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:24,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:24,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:28,120 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:04:29,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:30,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:34,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:35,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:39,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:40,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:44,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:45,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:49,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:50,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:54,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:04:55,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:04:58,121 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:04:59,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:01,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:04,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:06,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:09,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:11,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:14,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:16,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:19,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:21,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:24,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:25,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:05:25,496 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:05:25,496 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:05:25,497 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:05:26,162 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:05:26,497 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:28,122 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:05:29,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:32,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:32,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:05:32,223 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:05:32,223 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:05:32,223 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:05:33,166 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:05:34,166 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:05:34,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:37,516 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:39,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:42,517 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:44,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:47,518 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:49,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:52,519 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:54,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:05:57,521 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:05:58,123 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:05:59,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:03,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:04,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:08,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:09,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:13,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:14,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:18,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:19,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:23,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:24,233 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:28,124 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:06:29,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:29,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:34,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:34,237 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:39,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:39,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:44,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:44,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:44,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:06:44,284 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:06:44,284 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:06:44,284 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:06:45,193 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:06:49,239 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:49,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:50,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:06:50,990 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:06:50,991 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:06:50,991 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:06:51,195 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:06:54,196 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:06:54,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:06:55,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:06:58,125 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:06:59,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:01,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:04,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:06,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:09,248 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:11,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:14,250 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:16,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:19,251 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:21,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:24,252 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:26,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:28,126 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:07:29,253 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:32,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:34,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:37,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:39,255 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:42,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:44,256 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:47,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:49,257 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:52,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:54,261 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:07:57,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:07:58,127 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:07:59,262 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:03,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:03,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:08:03,271 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:08:03,272 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:08:03,272 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:08:04,221 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:08:04,263 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:08,273 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:09,265 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:09,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:08:09,984 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:08:09,985 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:08:09,985 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:08:10,224 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:08:12,225 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:08:14,241 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:14,266 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:19,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:19,267 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:24,243 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:24,268 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:28,128 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:08:29,269 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:30,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:34,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:35,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:39,271 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:40,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:44,272 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:45,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:49,273 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:50,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:54,274 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:08:55,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:08:58,129 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:08:59,275 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:01,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:04,276 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:06,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:09,280 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:11,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:14,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:16,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:19,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:21,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:21,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:09:21,763 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:09:21,763 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:09:21,764 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:09:22,252 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:09:24,283 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:26,765 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:28,130 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:09:28,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:09:28,479 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:09:28,479 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:09:28,480 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:09:29,254 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:09:29,284 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:30,255 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:09:32,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:34,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:37,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:39,286 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:42,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:44,287 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:47,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:49,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:52,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:54,289 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:09:57,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:09:58,131 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:09:59,291 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:02,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:04,292 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:07,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:09,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:12,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:14,294 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:17,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:19,295 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:22,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:24,296 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:27,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:28,132 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:10:29,298 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:33,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:34,299 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:38,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:39,300 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:41,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:10:41,221 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:10:41,222 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:10:41,222 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:10:41,281 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:10:43,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:44,301 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:47,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:10:47,942 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:10:47,942 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:10:47,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:10:48,284 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:10:49,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:49,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:50,285 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:10:54,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:10:54,303 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:10:58,134 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:10:59,304 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:00,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:04,305 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:05,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:09,306 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:10,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:14,307 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:15,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:19,308 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:20,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:24,309 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:25,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:28,135 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:11:29,310 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:31,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:34,311 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:36,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:39,312 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:41,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:44,312 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:46,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:49,313 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:51,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:54,314 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:56,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:11:58,135 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:11:59,315 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:11:59,503 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:11:59,504 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:11:59,504 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:11:59,505 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:12:00,312 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:12:01,505 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:04,316 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:06,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:12:06,220 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:12:06,220 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:12:06,221 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:12:06,314 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:12:07,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:08,315 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:12:09,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:12,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:14,318 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:17,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:19,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:22,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:24,320 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:27,495 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:28,137 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:12:29,321 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:33,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:34,322 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:38,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:39,323 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:43,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:44,324 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:48,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:49,325 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:53,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:54,326 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:12:58,138 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:12:59,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:12:59,327 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:04,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:04,328 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:09,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:09,329 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:14,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:14,330 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:16,905 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:13:16,906 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:13:16,906 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:13:16,906 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:13:17,341 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:13:19,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:19,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:23,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:13:23,631 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:13:23,631 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:13:23,632 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:13:24,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:24,344 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:13:25,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:26,345 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:13:28,139 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:13:29,333 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:31,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:34,333 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:36,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:39,334 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:41,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:44,335 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:46,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:49,336 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:51,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:54,337 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:13:56,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:13:58,140 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:13:59,338 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:02,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:04,339 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:07,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:09,340 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:12,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:14,341 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:17,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:19,342 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:22,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:24,343 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:27,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:28,141 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:14:29,344 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:33,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:34,345 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:35,675 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:14:35,677 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:14:35,677 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:14:35,678 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:14:36,369 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:14:38,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:39,346 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:42,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:14:42,404 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:14:42,404 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:14:42,405 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:14:43,372 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:14:44,346 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:44,372 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:14:44,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:49,347 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:49,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:54,348 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:14:54,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:14:58,142 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:14:59,349 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:00,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:04,350 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:05,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:09,351 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:10,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:14,352 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:15,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:19,353 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:20,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:24,354 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:25,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:28,143 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:15:29,355 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:31,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:34,356 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:36,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:39,357 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:41,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:44,358 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:46,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:49,360 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:51,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:54,361 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:15:54,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:15:54,822 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:15:54,822 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:15:54,823 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:15:55,398 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:15:56,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:15:58,144 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:15:59,361 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:01,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:16:01,532 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:16:01,532 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:16:01,533 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:16:02,400 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:16:02,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:04,362 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:04,401 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:16:07,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:09,363 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:12,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:14,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:17,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:19,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:22,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:24,366 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:27,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:28,145 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:16:29,367 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:33,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:34,368 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:38,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:39,369 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:43,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:44,370 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:48,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:49,371 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:53,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:54,372 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:16:58,146 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:16:59,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:16:59,373 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:04,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:04,374 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:09,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:09,375 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:12,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:17:12,431 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:17:12,431 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:17:12,432 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:17:13,427 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:17:14,376 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:14,432 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:19,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:17:19,149 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:17:19,149 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:17:19,150 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:17:19,382 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:19,430 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:17:20,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:20,430 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:17:24,383 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:25,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:28,147 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:17:29,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:31,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:34,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:36,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:39,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:41,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:44,387 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:46,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:49,388 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:51,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:54,389 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:17:56,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:17:58,148 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:17:59,390 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:02,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:04,390 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:07,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:09,391 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:12,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:14,392 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:17,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:19,393 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:22,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:24,394 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:27,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:28,148 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:18:29,395 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:31,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:18:31,879 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:18:31,880 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:18:31,880 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:18:32,457 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:18:32,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:34,397 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:37,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:38,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:18:38,609 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:18:38,609 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:18:38,609 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:18:39,398 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:39,460 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:18:40,461 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:18:43,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:44,399 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:48,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:49,400 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:53,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:54,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:18:58,149 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:18:59,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:18:59,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:04,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:04,403 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:09,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:09,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:14,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:14,405 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:19,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:19,406 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:24,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:24,407 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:28,151 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:19:29,408 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:30,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:34,409 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:35,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:39,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:40,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:44,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:45,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:49,412 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:50,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:50,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:19:50,819 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:19:50,820 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:19:50,820 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:19:51,487 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:19:54,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:19:55,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:19:57,523 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:19:57,524 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:19:57,524 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:19:57,525 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:19:58,152 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:19:58,490 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:19:58,490 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:19:59,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:01,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:04,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:06,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:09,416 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:11,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:14,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:16,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:19,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:21,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:24,419 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:26,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:28,152 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:20:29,421 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:32,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:34,422 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:37,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:39,423 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:42,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:44,424 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:47,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:49,425 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:52,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:54,426 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:20:57,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:20:58,153 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:20:59,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:03,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:04,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:08,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:09,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:21:09,088 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:21:09,088 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:21:09,089 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:21:09,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:09,515 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:21:14,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:14,429 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:15,805 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:21:15,807 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:21:15,807 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:21:15,807 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:21:16,518 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:21:16,519 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:21:19,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:20,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:24,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:25,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:28,154 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:21:29,432 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:30,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:34,433 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:35,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:39,434 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:40,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:44,435 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:45,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:49,436 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:50,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:54,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:21:55,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:21:58,155 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:21:59,438 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:01,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:04,439 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:06,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:09,440 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:11,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:14,441 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:16,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:19,444 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:21,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:24,445 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:26,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:22:26,046 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:22:26,046 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:22:26,046 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:22:26,544 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:22:27,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:28,156 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:22:29,446 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:32,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:32,747 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:22:32,748 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:22:32,748 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:22:32,749 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:22:33,547 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:22:34,447 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:35,548 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:22:38,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:39,448 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:43,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:44,449 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:48,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:49,450 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:53,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:54,451 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:22:58,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:22:58,157 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:22:59,452 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:03,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:04,453 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:08,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:09,454 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:13,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:14,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:18,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:19,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:23,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:24,457 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:28,158 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:23:29,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:29,458 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:34,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:34,459 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:39,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:39,460 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:43,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:23:43,845 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:23:43,846 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:23:43,846 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:23:44,461 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:44,576 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:23:44,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:49,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:49,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:50,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:23:50,550 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:23:50,551 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:23:50,551 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:23:50,578 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:23:53,579 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:23:54,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:23:55,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:23:58,159 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:23:59,464 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:01,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:04,464 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:06,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:09,465 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:11,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:14,466 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:16,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:19,467 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:21,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:24,468 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:26,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:28,160 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:24:29,470 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:32,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:34,470 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:37,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:39,471 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:42,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:44,472 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:47,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:49,473 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:52,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:54,474 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:24:57,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:24:58,161 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:24:59,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:02,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:25:02,477 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:25:02,477 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:25:02,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:02,478 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:25:02,605 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:25:04,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:07,479 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:09,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:25:09,185 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:25:09,185 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:25:09,185 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:25:09,478 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:09,608 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:25:11,609 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:25:13,426 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:14,479 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:18,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:19,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:23,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:24,481 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:28,162 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:25:29,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:29,482 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:34,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:34,483 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:39,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:39,484 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:44,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:44,485 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:49,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:49,486 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:54,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:25:54,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:25:58,163 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:25:59,488 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:00,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:04,489 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:05,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:09,490 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:10,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:14,491 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:15,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:19,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:20,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:20,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:26:20,219 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:26:20,219 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:26:20,220 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:26:20,634 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:26:24,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:25,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:26,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:26:26,925 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:26:26,925 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:26:26,926 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:26:27,637 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:26:28,164 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:26:29,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:29,637 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:26:31,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:34,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:36,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:39,495 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:41,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:44,496 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:46,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:49,497 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:51,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:54,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:26:56,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:26:58,165 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:26:59,499 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:02,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:04,500 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:07,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:09,501 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:12,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:14,502 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:17,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:19,503 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:22,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:24,504 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:27,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:28,166 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:27:29,505 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:33,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:34,506 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:38,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:38,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:27:38,706 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:27:38,707 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:27:38,707 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:27:39,507 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:39,664 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:27:43,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:44,508 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:45,422 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:27:45,424 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:27:45,424 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:27:45,425 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:27:45,666 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:27:47,667 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:27:49,509 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:49,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:54,511 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:27:54,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:27:58,167 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:27:59,512 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:00,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:04,513 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:05,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:09,514 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:10,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:14,515 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:15,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:19,515 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:20,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:24,516 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:25,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:28,168 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:28:29,517 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:31,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:34,518 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:36,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:39,519 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:41,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:44,520 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:46,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:49,521 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:51,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:54,522 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:28:56,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:28:56,337 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:28:56,339 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:28:56,339 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:28:56,339 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:28:56,692 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:28:58,169 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:28:59,523 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:02,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:03,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:29:03,063 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:29:03,063 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:29:03,064 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:29:03,695 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:29:04,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:05,695 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:29:07,300 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:09,525 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:12,301 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:14,526 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:17,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:19,528 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:22,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:24,529 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:27,303 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:28,170 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:29:29,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:33,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:34,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:38,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:39,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:43,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:44,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:48,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:49,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:53,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:54,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:29:58,171 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:29:59,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:29:59,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:04,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:04,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:09,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:09,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:14,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:14,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:30:14,432 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:30:14,432 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:30:14,433 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:30:14,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:14,722 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:30:19,434 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:19,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:21,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:30:21,140 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:30:21,140 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:30:21,141 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:30:21,724 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:30:23,725 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:30:24,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:25,386 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:28,172 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:30:29,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:31,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:34,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:36,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:39,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:41,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:44,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:46,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:49,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:51,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:54,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:30:56,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:30:58,173 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:30:59,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:02,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:04,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:07,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:09,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:12,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:14,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:17,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:19,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:22,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:24,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:27,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:28,174 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:31:29,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:33,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:31:33,066 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:31:33,067 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:31:33,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:33,068 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:31:33,752 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:31:34,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:39,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:39,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:31:39,799 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:31:39,799 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:31:39,800 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:31:40,754 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:31:41,755 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:31:44,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:44,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:49,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:49,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:54,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:54,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:31:58,175 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:31:59,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:31:59,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:04,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:04,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:09,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:09,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:14,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:14,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:19,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:19,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:24,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:24,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:28,176 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:32:29,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:30,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:34,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:35,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:39,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:40,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:44,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:45,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:49,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:50,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:51,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:32:51,221 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:32:51,221 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:32:51,222 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:32:51,782 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:32:54,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:55,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:32:57,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:32:57,932 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:32:57,933 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:32:57,933 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:32:58,176 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:32:58,784 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:32:59,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:32:59,785 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:33:00,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:04,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:05,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:09,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:10,412 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:14,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:15,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:19,578 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:20,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:24,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:25,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:28,177 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:33:29,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:31,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:34,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:36,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:39,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:41,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:44,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:46,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:49,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:51,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:54,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:33:56,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:33:58,178 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:33:59,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:02,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:04,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:07,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:09,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:10,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:34:10,168 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:34:10,168 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:34:10,169 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:34:10,812 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:34:13,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:14,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:16,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:34:16,885 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:34:16,885 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:34:16,886 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:34:17,815 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:34:19,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:19,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:19,816 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:34:24,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:24,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:28,179 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:34:29,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:29,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:34,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:34,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:39,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:39,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:44,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:44,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:49,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:49,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:54,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:54,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:34:58,184 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:34:59,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:34:59,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:04,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:04,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:09,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:09,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:14,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:14,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:19,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:19,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:24,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:24,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:28,181 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:35:28,423 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:35:28,424 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:35:28,424 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:35:28,426 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:35:28,841 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:35:29,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:29,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:34,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:34,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:35,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:35:35,158 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:35:35,158 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:35:35,159 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:35:35,844 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:35:37,845 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:35:39,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:40,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:44,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:45,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:49,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:50,403 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:54,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:35:55,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:35:58,182 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:35:59,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:01,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:04,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:06,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:09,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:11,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:14,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:16,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:19,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:21,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:24,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:26,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:28,185 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:36:29,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:32,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:34,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:37,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:39,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:42,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:44,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:47,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:47,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:36:47,690 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:36:47,690 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:36:47,691 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:36:47,871 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:36:49,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:52,692 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:54,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:36:54,404 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:36:54,404 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:36:54,404 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:36:54,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:36:54,874 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:36:55,874 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:36:58,184 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:36:58,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:36:59,631 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:03,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:04,632 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:08,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:09,633 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:13,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:14,634 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:18,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:19,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:23,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:24,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:28,185 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:37:29,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:29,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:34,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:34,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:39,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:39,639 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:44,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:44,640 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:49,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:49,641 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:54,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:37:54,642 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:37:58,186 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:37:59,643 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:00,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:04,644 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:05,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:05,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:38:05,933 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:38:05,933 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:38:05,934 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:38:06,900 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:38:09,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:10,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:12,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:38:12,646 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:38:12,647 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:38:12,647 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:38:12,903 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:38:13,903 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:38:14,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:16,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:19,647 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:21,894 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:24,648 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:26,895 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:28,187 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:38:29,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:32,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:34,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:37,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:39,651 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:42,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:44,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:47,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:49,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:52,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:54,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:38:57,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:38:58,188 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:38:59,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:03,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:04,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:08,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:09,660 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:13,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:14,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:18,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:19,662 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:23,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:24,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:39:24,286 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:39:24,287 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:39:24,287 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:39:24,663 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:24,932 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:39:28,189 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:39:29,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:29,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:31,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:39:31,014 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:39:31,015 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:39:31,015 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:39:31,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:39:31,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:39:34,253 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:34,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:39,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:39,666 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:44,255 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:44,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:49,256 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:49,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:54,257 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:39:54,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:39:58,190 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:39:59,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:00,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:04,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:05,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:09,673 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:10,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:14,674 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:15,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:19,675 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:20,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:24,676 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:25,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:28,191 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:40:29,677 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:31,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:34,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:36,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:39,679 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:41,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:42,305 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:40:42,306 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:40:42,307 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:40:42,307 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:40:42,962 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:40:44,680 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:46,308 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:49,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:40:49,032 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:40:49,033 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:40:49,033 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:40:49,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:49,966 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:40:49,966 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:40:52,287 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:54,682 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:40:57,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:40:58,192 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:40:59,683 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:03,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:04,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:08,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:09,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:13,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:14,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:18,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:19,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:23,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:24,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:28,193 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:41:29,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:29,690 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:34,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:34,691 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:39,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:39,692 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:44,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:44,693 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:49,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:49,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:54,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:41:54,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:41:58,194 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:41:59,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:00,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:01,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:42:01,195 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:42:01,195 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:42:01,195 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:42:01,993 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:42:04,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:05,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:07,916 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:42:07,917 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:42:07,917 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:42:07,918 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:42:07,996 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:42:09,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:09,997 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:42:11,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:14,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:16,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:19,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:21,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:24,701 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:26,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:28,195 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:42:29,702 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:31,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:34,703 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:36,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:39,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:41,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:44,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:46,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:49,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:51,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:54,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:42:56,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:42:58,196 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:42:59,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:02,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:04,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:07,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:09,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:12,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:14,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:17,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:18,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:43:18,775 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:43:18,776 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:43:18,776 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:43:19,026 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:43:19,712 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:22,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:24,713 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:25,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:43:25,525 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:43:25,525 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:43:25,526 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:43:26,029 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:43:28,030 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:43:28,197 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:43:28,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:29,720 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:33,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:34,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:38,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:39,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:43,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:44,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:48,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:49,723 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:53,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:54,724 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:43:58,197 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:43:59,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:43:59,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:04,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:04,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:09,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:09,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:14,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:14,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:19,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:19,729 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:24,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:24,730 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:28,198 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:44:29,732 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:30,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:34,733 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:35,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:36,837 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:44:36,839 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:44:36,839 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:44:36,839 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:44:37,057 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:44:39,734 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:40,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:43,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:44:43,562 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:44:43,562 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:44:43,563 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:44:44,060 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:44:44,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:46,061 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:44:46,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:49,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:51,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:54,742 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:44:56,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:44:58,199 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:44:59,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:02,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:04,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:07,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:09,745 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:12,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:14,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:17,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:19,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:22,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:24,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:27,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:28,200 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:45:29,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:33,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:34,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:38,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:39,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:43,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:44,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:48,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:49,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:53,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:54,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:45:55,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:45:55,123 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:45:55,124 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:45:55,124 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:45:56,087 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:45:58,201 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:45:59,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:45:59,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:01,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:46:01,839 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:46:01,840 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:46:01,840 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:46:02,089 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:46:04,090 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:46:04,759 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:05,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:09,760 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:10,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:14,761 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:15,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:19,762 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:20,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:24,763 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:25,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:28,202 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:46:29,764 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:30,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:34,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:35,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:39,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:40,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:44,768 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:45,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:49,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:50,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:54,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:46:55,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:46:58,203 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:46:59,770 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:01,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:04,771 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:06,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:09,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:11,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:14,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:47:14,411 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:47:14,411 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:47:14,412 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:47:14,773 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:15,118 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:47:16,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:19,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:21,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:47:21,152 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:47:21,152 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:47:21,153 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:47:21,416 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:22,121 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:47:23,122 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:47:24,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:26,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:28,204 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:47:29,776 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:32,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:34,777 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:37,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:39,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:42,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:44,779 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:47,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:49,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:52,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:54,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:47:57,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:47:58,208 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:47:59,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:03,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:04,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:08,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:09,784 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:13,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:14,785 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:18,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:19,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:23,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:24,787 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:28,206 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:48:29,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:29,788 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:32,379 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:48:32,380 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:48:32,380 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:48:32,382 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:48:33,149 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:48:34,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:34,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:39,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:48:39,114 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:48:39,115 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:48:39,115 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:48:39,151 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:48:39,790 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:40,362 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:41,152 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:48:44,792 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:45,363 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:49,793 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:50,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:54,794 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:48:55,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:48:58,206 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:48:59,795 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:01,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:04,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:06,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:09,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:11,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:14,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:16,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:19,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:21,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:24,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:26,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:28,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:49:29,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:32,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:34,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:37,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:39,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:42,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:44,809 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:47,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:49,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:49:49,651 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:49:49,651 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:49:49,652 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:49:49,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:50,180 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:49:52,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:54,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:49:56,375 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:49:56,376 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:49:56,376 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:49:56,377 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:49:57,182 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:49:57,763 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:49:58,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:49:59,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:49:59,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:03,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:04,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:08,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:09,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:13,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:14,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:18,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:19,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:23,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:24,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:28,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:50:29,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:29,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:34,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:34,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:39,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:39,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:44,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:44,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:49,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:49,825 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:54,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:50:54,826 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:50:58,210 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:50:59,828 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:00,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:04,829 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:05,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:07,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:51:07,415 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:51:07,415 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:51:07,416 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:51:08,210 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:51:09,830 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:10,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:14,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:51:14,174 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:51:14,174 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:51:14,175 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:51:14,213 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:51:14,831 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:16,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:17,214 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:51:19,832 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:21,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:24,833 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:26,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:28,211 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:51:29,834 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:32,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:34,835 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:37,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:39,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:42,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:44,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:47,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:49,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:52,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:54,842 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:51:57,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:51:58,212 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:51:59,843 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:03,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:04,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:08,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:09,846 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:13,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:14,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:18,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:19,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:23,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:24,849 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:25,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:52:25,809 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:52:25,809 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:52:25,810 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:52:26,240 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:52:28,213 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:52:29,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:29,850 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:32,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:52:32,549 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:52:32,549 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:52:32,550 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:52:33,242 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:52:34,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:34,851 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:35,243 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:52:39,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:39,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:44,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:44,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:49,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:49,854 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:54,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:52:54,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:52:58,214 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:52:59,856 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:00,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:04,857 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:05,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:09,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:10,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:14,859 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:15,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:19,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:20,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:24,861 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:25,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:28,215 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:53:29,862 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:31,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:34,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:36,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:39,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:41,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:44,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:53:44,753 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:53:44,754 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:53:44,754 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:53:44,866 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:45,269 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:53:46,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:49,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:51,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:53:51,500 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:53:51,500 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:53:51,501 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:53:51,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:52,272 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:53:53,273 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:53:54,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:53:56,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:53:58,216 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:53:59,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:02,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:04,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:07,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:09,871 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:12,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:14,872 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:17,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:19,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:22,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:24,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:27,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:28,217 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:54:29,874 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:33,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:34,875 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:38,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:39,876 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:43,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:44,877 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:48,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:49,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:53,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:54,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:54:58,217 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:54:59,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:54:59,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:03,379 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:55:03,380 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:55:03,380 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:55:03,381 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:55:04,301 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:55:04,381 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:04,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:09,382 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:09,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:10,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:55:10,090 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:55:10,090 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:55:10,090 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:55:10,304 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:55:11,304 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:55:14,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:15,330 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:19,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:20,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:24,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:25,332 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:28,218 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:55:29,890 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:31,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:34,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:36,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:39,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:41,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:44,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:46,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:49,894 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:51,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:54,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:55:56,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:55:58,219 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:55:59,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:02,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:04,897 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:07,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:09,898 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:12,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:14,900 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:17,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:19,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:22,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:22,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:56:22,755 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:56:22,755 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:56:22,756 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:56:23,333 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:56:24,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:27,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:28,220 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:56:29,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:56:29,481 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:56:29,482 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:56:29,482 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:56:29,903 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:30,336 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:56:31,336 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:56:33,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:34,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:38,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:39,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:43,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:44,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:48,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:49,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:53,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:54,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:56:58,222 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:56:59,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:56:59,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:04,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:04,910 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:09,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:09,911 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:14,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:14,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:19,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:19,913 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:24,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:24,914 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:28,222 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:57:29,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:30,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:34,916 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:35,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:39,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:40,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:42,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:57:42,037 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:57:42,037 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:57:42,038 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:57:42,365 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:57:44,918 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:46,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:48,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:57:48,767 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:57:48,768 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:57:48,768 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:57:49,368 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:57:49,919 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:51,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:51,369 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:57:54,920 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:57:56,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:57:58,223 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:57:59,921 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:01,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:04,922 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:06,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:09,923 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:11,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:14,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:16,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:19,926 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:21,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:24,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:26,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:28,224 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:58:29,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:32,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:34,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:37,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:39,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:42,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:44,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:47,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:49,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:52,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:54,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:58:57,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:58:58,225 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:58:59,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:00,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:59:00,575 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:59:00,576 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:59:00,576 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:59:01,396 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:59:02,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:04,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:07,309 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 03:59:07,310 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 03:59:07,311 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 03:59:07,311 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 03:59:07,398 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 03:59:07,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:09,399 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 03:59:09,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:12,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:14,937 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:17,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:19,938 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:22,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:24,939 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:27,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:28,226 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:59:29,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:33,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:34,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:38,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:39,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:43,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:44,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:48,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:49,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:53,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:54,945 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 03:59:58,227 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 03:59:59,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 03:59:59,946 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:04,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:04,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:09,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:09,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:14,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:14,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:19,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 04:00:19,102 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 04:00:19,103 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 04:00:19,103 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 04:00:19,425 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 04:00:19,950 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:20,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:24,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:25,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:25,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 04:00:25,840 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 04:00:25,840 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 04:00:25,841 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 04:00:26,427 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 04:00:27,428 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:00:28,228 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:00:29,953 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:30,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:34,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:35,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:39,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:40,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:44,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:45,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:49,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:50,232 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:54,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:00:55,233 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:00:58,269 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:00:59,959 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:00,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:04,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:05,271 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:09,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:10,272 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:10,921 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 04:01:10,922 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 04:01:10,923 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 04:01:10,923 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 04:01:11,444 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 04:01:14,961 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:15,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:17,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 04:01:17,669 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 04:01:17,669 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 04:01:17,669 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 04:01:17,925 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history -2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send():378] send: metric -2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send():378] send: history -2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record -2024-03-28 04:01:17,931 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end -2024-03-28 04:01:17,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause -2024-03-28 04:01:17,932 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-28 04:01:17,932 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor -2024-03-28 04:01:17,932 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-28 04:01:17,932 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-28 04:01:17,933 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor -2024-03-28 04:01:17,933 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor -2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor -2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor -2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor -2024-03-28 04:01:17,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume -2024-03-28 04:01:17,951 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-28 04:01:17,951 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 04:01:17,952 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 04:01:17,952 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:01:17,953 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 04:01:17,953 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 04:01:17,954 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 04:01:17,955 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 04:01:17,956 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 04:01:18,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause -2024-03-28 04:01:18,130 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-28 04:01:18,130 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor -2024-03-28 04:01:18,131 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor -2024-03-28 04:01:18,131 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor -2024-03-28 04:01:18,131 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-28 04:01:18,132 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-28 04:01:18,132 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor -2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor -2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor -2024-03-28 04:01:18,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume -2024-03-28 04:01:18,146 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-28 04:01:18,146 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 04:01:18,146 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 04:01:18,147 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 04:01:18,147 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 04:01:18,147 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:01:18,149 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 04:01:18,149 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 04:01:18,150 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 04:01:18,447 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json -2024-03-28 04:01:19,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause -2024-03-28 04:01:19,766 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-28 04:01:19,766 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor -2024-03-28 04:01:19,766 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-28 04:01:19,767 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-28 04:01:19,767 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-28 04:01:19,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor -2024-03-28 04:01:19,768 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor -2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor -2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor -2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor -2024-03-28 04:01:19,779 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:01:19,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:21,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:24,969 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:26,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:29,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:31,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:34,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:36,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:39,972 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:41,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:44,973 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:46,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:48,458 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml -2024-03-28 04:01:49,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:52,988 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:54,975 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:01:57,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:01:59,976 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:02,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:04,977 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:07,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:09,978 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:12,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:14,979 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:17,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:19,980 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:22,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:24,981 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:27,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:29,982 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:32,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:34,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:37,995 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:39,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:42,996 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:44,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:47,997 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:49,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:52,998 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:54,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:02:57,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:02:59,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:03,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:04,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:08,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:09,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:13,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:14,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:18,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:19,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:23,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:24,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:28,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:29,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:33,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:34,995 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:38,006 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:39,996 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:43,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:44,997 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:48,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:49,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:53,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:03:55,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:03:58,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:00,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:03,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:05,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:08,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:10,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:13,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:15,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:18,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:20,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:23,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:25,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:28,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:30,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:33,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:35,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:38,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:40,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:43,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:45,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:48,019 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:50,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:53,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:04:55,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:04:58,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:00,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:03,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:05,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:08,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:10,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:13,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:15,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:18,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:20,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:23,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:25,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:28,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:30,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:33,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:35,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:38,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:40,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:43,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:45,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:48,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:50,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:53,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:05:55,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:05:58,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:00,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:03,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:05,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:08,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:10,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:13,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:15,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:18,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:20,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:23,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:25,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:28,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:30,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:33,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:35,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:38,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:40,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:43,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:45,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:48,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:50,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:53,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:06:55,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:06:58,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:00,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:03,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:05,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:08,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:10,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:13,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:15,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:18,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:20,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:23,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:25,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:28,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:30,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:33,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:35,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:38,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:40,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:43,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:45,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:48,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:50,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:53,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:07:55,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:07:58,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:00,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:03,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:05,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:08,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:10,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:13,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:15,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:18,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:20,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:23,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:25,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:28,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:30,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:33,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:35,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:38,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:40,059 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:43,059 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:45,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:48,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:50,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:53,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:08:55,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:08:58,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:00,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:03,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:05,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:08,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:10,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:13,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:15,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:18,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:20,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:23,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:25,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:28,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:30,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:33,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:35,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:40,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:43,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:45,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:48,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:50,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:53,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:09:55,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:09:58,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:00,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:03,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:05,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:08,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:10,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:13,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:15,077 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:18,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:20,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:23,077 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:25,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:28,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:30,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:33,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:35,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:38,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:40,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:43,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:45,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:48,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:50,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:53,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:10:55,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:10:58,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:00,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:03,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:05,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:08,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:10,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:13,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:15,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:18,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:20,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:23,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:25,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:28,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:30,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:33,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:35,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:38,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:40,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:43,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:45,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:48,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:50,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:53,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:11:55,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:11:58,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:00,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:03,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:05,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:08,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:10,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:13,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:15,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:18,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:20,102 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:23,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:25,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:28,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:30,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:33,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:35,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:38,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:40,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:43,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:45,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:48,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:50,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:53,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:12:55,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:12:58,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:00,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:03,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:05,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:08,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:10,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:13,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:15,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:18,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:20,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:23,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:25,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:28,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:30,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:33,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:35,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:38,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:40,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:43,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:45,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:48,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:50,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:53,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:13:55,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:13:58,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:00,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:03,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:05,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:08,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:10,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:13,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:15,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:18,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:20,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:23,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:25,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:28,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:30,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:33,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:35,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:38,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:40,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:43,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:45,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:48,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:50,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:53,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:14:55,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:14:58,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:00,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:03,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:05,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:08,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:10,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:13,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:15,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:18,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:20,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:23,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:25,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:28,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:30,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:33,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:35,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:38,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:40,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:43,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:45,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:48,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:50,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:53,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:15:55,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:15:58,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:00,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:03,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:05,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:08,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:10,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:13,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:15,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:18,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:20,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:23,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:25,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:28,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:30,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:33,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:35,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:38,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:40,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:43,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:45,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:48,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:50,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:53,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:16:55,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:16:58,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:00,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:03,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:05,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:08,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:10,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:13,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:15,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:18,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:20,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:23,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:25,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:28,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:30,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:33,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:35,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:38,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:40,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:43,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:45,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:48,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:50,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:53,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:17:55,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:17:58,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:00,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:03,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:05,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:08,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:10,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:13,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:15,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:18,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:20,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:23,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:25,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:28,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:30,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:33,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:35,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:38,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:40,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:43,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:45,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:48,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:50,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:53,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:18:55,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:18:58,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:00,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:03,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:05,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:08,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:10,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:13,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:15,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:18,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:20,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:23,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:25,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:28,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:30,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:33,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:35,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:38,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:40,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:43,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:45,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:48,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:50,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:53,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:19:55,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:19:58,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:00,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:03,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:05,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:08,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:10,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:13,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:15,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:18,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:20,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:23,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:25,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:28,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:30,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:33,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:35,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:38,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:40,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:43,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:45,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:48,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:50,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:53,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:20:55,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:20:58,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:00,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:03,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:05,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:08,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:10,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:13,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:15,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:18,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:20,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:23,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:25,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:28,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:30,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:33,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:35,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:38,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:40,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:43,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:45,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:48,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:50,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:53,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:21:55,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:21:58,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:00,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:03,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume -2024-03-28 04:22:03,057 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-28 04:22:03,057 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 04:22:03,057 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 04:22:03,058 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 04:22:03,059 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 04:22:03,059 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 04:22:03,060 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 04:22:03,061 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 04:22:03,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause -2024-03-28 04:22:03,088 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-28 04:22:03,088 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor -2024-03-28 04:22:03,088 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-28 04:22:03,088 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-28 04:22:03,089 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-28 04:22:03,089 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor -2024-03-28 04:22:03,091 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor -2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor -2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor -2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor -2024-03-28 04:22:03,101 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:22:04,102 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:05,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:09,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:10,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:14,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:15,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:19,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:20,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:24,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:25,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:29,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:30,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:34,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:35,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:39,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:40,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:44,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:45,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:45,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume -2024-03-28 04:22:45,412 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-28 04:22:45,412 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 04:22:45,412 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 04:22:45,412 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 04:22:45,413 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 04:22:45,415 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 04:22:45,416 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 04:22:45,417 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 04:22:46,969 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:22:48,970 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:22:49,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:50,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:54,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:22:55,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:22:58,974 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:22:59,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:23:00,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:23:00,974 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:23:02,975 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log -2024-03-28 04:23:04,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:23:05,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:23:09,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:23:10,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive -2024-03-28 04:23:10,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause -2024-03-28 04:23:10,941 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-03-28 04:23:10,941 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor -2024-03-28 04:23:10,941 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-28 04:23:10,941 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-03-28 04:23:10,941 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor -2024-03-28 04:23:10,942 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-03-28 04:23:10,942 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor -2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor -2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor -2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor -2024-03-28 04:23:10,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume -2024-03-28 04:23:10,955 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread -2024-03-28 04:23:10,955 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor -2024-03-28 04:23:10,956 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-28 04:23:10,956 DEBUG SenderThread:118 [sender.py:send():378] send: stats -2024-03-28 04:23:10,956 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring -2024-03-28 04:23:10,957 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring -2024-03-28 04:23:10,958 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring -2024-03-28 04:23:10,958 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring -2024-03-28 04:23:10,959 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring -2024-03-28 04:23:14,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report -2024-03-28 04:23:15,232 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:30:04,910 INFO StreamThr :118 [internal.py:wandb_internal():86] W&B internal server running at pid: 118, started at: 2024-03-28 13:30:04.909963 +2024-03-28 13:30:04,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status +2024-03-28 13:30:05,251 INFO WriterThread:118 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/run-d0uv7bu9.wandb +2024-03-28 13:30:05,251 DEBUG SenderThread:118 [sender.py:send():378] send: header +2024-03-28 13:30:05,257 DEBUG SenderThread:118 [sender.py:send():378] send: run +2024-03-28 13:30:05,423 INFO SenderThread:118 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files +2024-03-28 13:30:05,423 INFO SenderThread:118 [sender.py:_start_run_threads():1099] run started: d0uv7bu9 with start time 1711632604.911076 +2024-03-28 13:30:05,425 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: check_version +2024-03-28 13:30:05,425 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: check_version +2024-03-28 13:30:05,513 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: run_start +2024-03-28 13:30:05,526 DEBUG HandlerThread:118 [system_info.py:__init__():26] System info init +2024-03-28 13:30:05,526 DEBUG HandlerThread:118 [system_info.py:__init__():41] System info init done +2024-03-28 13:30:05,526 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 13:30:05,526 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 13:30:05,527 INFO HandlerThread:118 [system_monitor.py:probe():214] Collecting system info +2024-03-28 13:30:05,527 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 13:30:05,527 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 13:30:05,528 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 13:30:05,529 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 13:30:05,531 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 13:30:05,542 DEBUG HandlerThread:118 [system_info.py:probe():150] Probing system +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_info.py:probe():198] Probing system done +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-28T13:30:05.542748', 'startedAt': '2024-03-28T13:30:04.903309', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'e1d09984cc69', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.202, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5533.654685974121}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-03-28 13:30:05,545 INFO HandlerThread:118 [system_monitor.py:probe():224] Finished collecting system info +2024-03-28 13:30:05,545 INFO HandlerThread:118 [system_monitor.py:probe():227] Publishing system info +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-03-28 13:30:06,425 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/conda-environment.yaml +2024-03-28 13:31:01,440 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/conda-environment.yaml +2024-03-28 13:31:01,973 DEBUG HandlerThread:118 [system_info.py:_save_conda():219] Saving conda packages done +2024-03-28 13:31:01,973 INFO HandlerThread:118 [system_monitor.py:probe():229] Finished publishing system info +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: python_packages +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,987 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: python_packages +2024-03-28 13:31:01,990 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,990 DEBUG SenderThread:118 [sender.py:send():378] send: config +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:01,992 WARNING SenderThread:118 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: stop_status +2024-03-28 13:31:02,085 DEBUG SenderThread:118 [sender.py:send():378] send: files +2024-03-28 13:31:02,085 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now +2024-03-28 13:31:02,303 INFO wandb-upload_0:118 [upload_job.py:push():131] Uploaded file /tmp/tmpe9_g8s3rwandb/cmzqhrrs-wandb-metadata.json +2024-03-28 13:31:02,440 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-metadata.json +2024-03-28 13:31:02,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/requirements.txt +2024-03-28 13:31:02,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:04,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:05,531 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 13:31:05,533 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:31:05,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:06,442 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 13:31:06,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:10,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:11,525 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:15,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:16,526 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:20,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:21,527 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:25,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:26,528 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:26,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:31:26,947 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:31:27,450 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:31:30,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:31,529 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:31,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:31:31,810 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:31:32,452 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:31:34,453 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:35,534 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:31:36,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:36,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:37,455 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 13:31:41,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:41,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:46,532 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:46,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:51,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:51,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:56,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:56,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:01,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:01,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:05,535 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:32:06,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:07,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:11,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:12,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:16,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:17,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:21,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:22,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:24,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:32:24,842 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:32:24,842 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:32:24,844 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:32:25,475 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:32:26,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:27,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:29,963 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:32:29,964 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:32:29,964 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:32:29,965 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:32:30,477 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:32:31,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:32,478 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:32:33,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:35,535 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:32:36,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:38,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:41,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:43,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:46,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:48,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:51,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:53,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:56,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:58,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:01,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:03,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:05,537 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:33:06,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:09,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:11,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:14,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:16,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:19,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:21,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:23,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:33:23,104 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:33:23,104 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:33:23,107 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:33:23,498 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:33:25,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:26,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:28,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:33:28,130 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:33:28,131 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:33:28,131 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:33:28,500 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:33:30,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:30,501 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:33:31,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:35,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:35,538 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:33:36,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:40,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:41,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:45,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:46,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:50,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:51,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:55,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:56,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:00,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:01,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:05,539 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:34:06,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:06,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:11,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:11,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:16,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:16,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:21,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:21,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:22,264 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:34:22,265 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:34:22,265 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:34:22,267 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:34:22,521 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:34:26,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:27,247 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:34:27,248 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:34:27,248 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:34:27,249 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:27,249 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:34:27,523 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:34:28,523 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:34:31,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:32,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:35,539 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:34:36,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:37,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:41,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:42,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:46,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:47,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:51,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:52,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:56,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:57,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:01,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:02,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:05,540 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:35:06,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:08,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:11,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:13,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:16,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:18,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:21,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:21,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:35:21,861 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:35:21,862 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:35:21,862 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:35:22,545 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:35:23,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:26,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:26,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:35:26,870 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:35:26,870 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:35:26,871 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:35:27,547 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:35:28,548 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:35:29,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:31,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:34,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:35,541 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:35:36,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:39,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:41,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:44,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:46,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:49,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:51,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:54,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:56,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:59,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:01,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:04,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:05,542 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:36:06,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:10,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:11,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:15,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:16,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:20,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:20,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:36:20,889 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:36:20,889 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:36:20,889 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:36:21,568 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:36:21,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:25,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:25,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:36:25,893 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:36:25,894 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:36:25,894 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:36:26,570 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:36:26,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:28,570 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:36:31,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:31,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:35,543 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:36:36,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:36,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:41,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:41,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:46,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:46,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:51,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:51,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:56,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:56,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:01,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:01,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:05,544 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:37:06,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:07,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:11,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:12,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:16,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:17,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:19,508 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:37:19,509 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:37:19,509 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:37:19,509 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:37:19,589 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:37:21,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:23,510 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:24,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:37:24,500 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:37:24,500 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:37:24,501 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:37:24,591 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:37:26,592 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:37:26,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:28,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:31,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:33,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:35,545 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:37:36,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:39,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:41,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:44,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:46,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:49,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:51,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:54,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:56,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:59,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:01,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:04,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:05,546 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:38:06,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:10,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:11,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:15,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:16,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:18,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:38:18,950 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:38:18,950 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:38:18,951 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:38:19,612 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:38:20,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:21,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:23,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:38:23,948 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:38:23,948 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:38:23,949 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:38:24,614 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:38:26,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:26,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:26,614 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:38:31,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:31,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:35,547 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:38:36,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:36,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:41,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:41,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:46,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:46,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:51,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:51,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:56,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:56,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:01,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:01,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:05,548 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:39:06,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:07,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:11,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:12,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:16,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:17,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:39:17,085 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:39:17,085 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:39:17,086 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:39:17,632 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:39:18,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:21,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:22,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:39:22,088 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:39:22,088 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:39:22,088 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:39:22,634 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:39:23,436 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:24,635 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:39:26,625 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:28,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:31,626 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:33,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:35,556 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:39:36,627 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:38,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:41,628 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:43,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:46,629 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:48,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:51,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:53,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:56,631 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:58,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:01,632 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:03,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:05,550 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:40:06,633 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:09,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:11,634 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:14,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:16,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:40:16,235 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:40:16,235 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:40:16,237 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:40:16,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:16,654 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:40:20,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:21,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:40:21,229 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:40:21,230 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:40:21,230 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:40:21,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:21,656 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:40:23,657 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:40:25,460 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:26,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:30,461 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:31,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:35,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:35,551 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:40:36,639 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:40,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:41,640 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:45,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:46,641 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:50,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:51,642 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:55,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:56,643 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:00,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:01,644 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:05,555 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:41:06,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:06,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:11,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:11,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:14,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:41:14,723 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:41:14,724 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:41:14,724 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:41:15,676 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:41:16,647 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:16,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:19,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:41:19,722 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:41:19,723 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:41:19,723 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:41:20,678 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:41:21,649 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:21,678 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:41:21,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:26,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:26,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:31,651 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:31,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:35,554 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:41:36,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:37,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:41,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:42,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:46,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:47,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:51,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:52,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:56,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:57,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:01,657 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:02,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:05,555 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:42:06,658 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:08,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:11,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:13,237 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:42:13,239 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:42:13,239 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:42:13,239 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:42:13,698 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:42:14,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:16,660 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:18,235 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:42:18,236 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:42:18,237 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:42:18,237 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:42:18,700 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:42:19,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:21,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:21,701 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:42:24,476 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:26,662 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:29,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:31,663 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:34,478 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:35,556 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:42:36,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:39,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:41,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:44,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:46,666 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:49,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:51,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:54,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:56,668 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:59,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:01,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:04,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:05,557 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:43:06,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:10,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:11,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:43:11,671 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:43:11,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:11,672 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:43:11,672 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:43:11,720 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:43:15,673 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:16,677 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:16,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:43:16,682 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:43:16,682 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:43:16,683 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:43:16,721 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:43:19,723 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:43:20,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:21,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:25,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:26,679 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:30,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:31,680 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:35,558 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:43:36,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:36,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:41,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:41,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:46,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:46,682 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:51,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:51,683 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:56,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:56,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:01,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:01,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:05,559 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:44:06,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:07,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:10,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:44:10,658 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:44:10,658 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:44:10,658 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:44:10,741 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:44:11,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:12,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:15,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:44:15,670 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:44:15,671 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:44:15,671 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:44:15,743 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:44:16,693 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:17,744 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:44:17,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:21,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:22,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:26,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:27,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:31,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:32,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:35,559 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:44:36,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:38,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:41,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:43,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:46,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:48,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:51,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:53,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:56,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:58,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:01,701 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:03,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:05,560 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:45:06,702 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:09,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:45:09,291 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:45:09,292 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:45:09,292 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:09,292 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:45:09,762 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:45:11,703 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:14,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:14,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:45:14,303 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:45:14,304 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:45:14,304 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:45:14,764 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:45:16,704 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:17,766 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:45:19,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:21,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:24,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:26,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:29,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:31,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:34,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:35,561 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:45:36,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:40,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:41,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:45,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:46,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:50,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:51,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:55,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:56,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:00,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:01,712 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:05,562 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:46:06,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:06,713 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:08,397 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:46:08,399 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:46:08,399 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:46:08,399 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:46:08,784 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:46:11,714 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:12,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:13,399 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:46:13,400 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:46:13,401 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:46:13,401 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:46:13,786 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:46:15,787 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:46:16,715 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:17,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:21,716 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:22,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:26,717 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:27,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:31,717 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:32,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:35,563 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:46:36,718 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:38,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:41,719 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:43,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:46,720 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:48,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:51,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:53,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:56,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:58,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:01,723 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:03,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:05,564 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:47:06,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:47:06,431 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:47:06,432 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:47:06,432 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:47:06,724 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:06,805 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:47:09,433 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:11,420 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:47:11,422 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:47:11,422 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:47:11,423 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:47:11,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:11,807 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:47:13,808 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:47:14,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:16,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:19,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:21,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:24,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:26,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:29,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:31,729 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:34,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:35,565 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:47:36,730 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:40,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:41,731 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:45,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:46,732 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:50,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:51,733 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:55,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:56,734 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:00,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:01,735 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:04,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:48:04,673 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:48:04,674 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:48:04,674 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:48:04,826 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:48:05,566 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:48:06,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:06,736 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:09,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:48:09,690 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:48:09,691 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:48:09,691 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:48:09,827 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:48:11,737 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:11,828 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:48:11,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:16,738 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:16,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:21,739 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:21,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:26,740 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:26,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:31,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:31,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:35,567 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:48:36,742 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:37,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:41,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:42,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:46,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:47,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:51,745 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:52,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:56,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:57,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:01,747 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:02,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:04,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:49:04,962 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:49:04,962 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:49:04,963 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:49:05,568 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:49:05,847 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:49:06,748 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:08,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:09,967 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:49:09,969 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:49:09,969 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:49:09,970 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:49:10,849 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:49:11,749 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:11,849 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:49:14,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:16,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:19,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:21,751 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:24,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:26,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:29,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:31,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:34,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:35,569 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:49:36,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:39,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:41,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:44,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:46,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:49,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:51,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:54,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:56,759 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:59,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:01,764 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:03,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:50:03,235 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:50:03,235 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:50:03,236 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:50:03,871 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:50:05,236 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:05,570 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:50:06,765 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:08,260 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:50:08,261 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:50:08,261 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:50:08,262 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:50:08,873 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:50:09,873 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:50:10,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:11,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:15,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:16,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:20,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:21,768 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:25,532 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:26,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:30,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:31,770 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:35,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:35,571 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:50:36,771 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:40,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:41,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:45,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:46,773 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:50,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:51,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:55,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:56,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:00,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:01,776 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:02,964 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:51:02,965 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:51:02,966 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:51:02,966 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:51:03,892 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:51:05,572 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:51:06,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:06,777 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:07,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:51:07,956 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:51:07,956 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:51:07,957 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:51:08,895 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:51:09,895 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:51:11,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:12,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:16,779 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:17,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:21,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:22,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:26,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:27,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:31,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:32,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:35,578 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:51:36,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:37,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:41,784 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:42,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:46,785 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:47,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:51,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:52,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:56,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:57,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:01,787 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:02,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:02,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:52:02,943 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:52:02,943 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:52:02,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:52:03,915 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:52:05,574 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:52:06,788 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:07,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:52:07,935 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:52:07,936 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:52:07,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:07,939 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:52:08,916 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:52:09,917 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:52:11,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:13,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:16,790 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:18,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:21,791 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:23,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:26,792 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:28,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:31,793 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:33,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:35,575 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:52:36,794 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:38,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:41,795 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:43,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:46,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:48,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:51,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:53,578 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:56,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:58,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:01,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:01,918 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:53:01,919 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:53:01,919 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:53:01,921 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:53:01,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:53:03,922 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:05,575 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:53:06,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:06,919 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:53:06,921 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:53:06,921 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:53:06,922 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:53:06,936 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:53:09,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:09,938 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:53:11,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:14,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:16,803 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:19,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:21,804 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:24,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:26,805 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:29,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:31,806 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:34,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:35,576 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:53:36,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:39,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:41,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:44,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:46,809 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:49,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:51,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:54,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:56,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:59,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:01,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:54:01,413 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:54:01,413 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:54:01,414 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:54:01,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:01,959 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:54:05,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:05,577 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:54:06,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:54:06,412 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:54:06,412 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:54:06,412 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:54:06,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:06,961 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:54:07,961 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:54:10,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:11,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:15,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:16,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:20,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:21,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:25,657 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:26,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:30,658 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:31,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:35,578 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:54:36,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:36,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:41,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:41,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:46,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:46,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:51,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:51,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:56,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:56,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:00,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:00,612 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:00,613 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:00,613 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:55:00,982 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:55:01,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:01,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:05,579 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:55:05,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:05,625 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:05,626 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:05,626 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:55:05,984 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:55:06,830 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:06,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:07,985 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:55:11,831 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:11,866 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:16,832 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:16,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:21,833 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:21,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:26,834 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:26,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:31,835 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:31,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:35,580 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:55:36,836 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:37,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:41,837 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:42,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:46,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:47,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:51,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:52,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:56,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:57,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:59,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:59,024 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:59,025 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:59,025 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:00,004 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:56:01,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:03,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:04,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:56:04,048 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:56:04,048 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:56:04,049 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:05,007 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:56:05,581 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:56:06,007 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:56:06,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:08,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:11,843 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:13,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:16,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:18,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:21,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:23,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:26,846 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:28,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:31,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:33,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:35,582 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:56:36,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:39,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:41,849 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:44,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:46,850 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:49,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:51,851 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:54,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:56,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:58,473 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:56:58,474 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:56:58,474 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:56:58,475 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:59,026 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:57:00,476 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:01,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:03,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:57:03,482 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:57:03,482 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:57:03,482 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:57:04,028 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:57:05,583 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:57:05,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:06,029 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:57:06,854 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:10,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:11,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:15,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:16,856 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:20,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:21,857 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:25,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:26,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:30,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:31,859 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:35,584 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:57:36,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:36,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:41,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:41,861 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:46,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:46,862 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:51,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:51,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:56,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:56,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:57,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:57:57,902 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:57:57,902 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:57:57,903 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:57:58,048 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:01,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:01,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:02,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:58:02,893 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:58:02,894 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:58:02,894 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:58:03,050 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:04,051 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:58:05,585 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:58:06,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:07,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:11,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:12,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:16,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:17,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:21,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:22,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:26,871 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:27,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:31,872 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:32,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:35,586 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:58:36,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:38,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:41,874 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:43,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:46,875 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:48,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:51,876 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:53,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:56,877 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:57,668 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:58:57,669 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:58:57,669 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:58:57,670 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:58:58,072 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:58,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:01,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:02,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:59:02,666 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:59:02,667 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:59:02,667 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:59:03,074 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:59:03,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:04,074 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:59:05,587 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:59:06,879 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:09,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:11,880 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:14,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:16,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:19,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:21,882 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:24,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:26,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:29,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:31,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:34,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:35,588 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:59:36,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:40,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:41,887 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:45,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:46,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:50,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:51,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:55,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:56,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:59:56,653 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:59:56,653 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:59:56,653 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:59:56,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:57,094 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:00,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:01,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:00:01,657 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:00:01,658 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:00:01,658 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:00:01,890 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:02,095 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:04,096 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:00:05,589 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:00:06,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:06,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:11,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:11,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:16,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:16,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:21,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:21,895 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:26,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:26,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:31,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:31,897 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:35,590 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:00:36,898 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:37,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:41,899 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:42,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:46,900 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:47,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:51,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:52,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:55,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:00:55,488 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:00:55,489 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:00:55,489 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:00:56,115 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:56,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:58,490 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:00,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:00,494 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:00,494 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:00,495 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:01:01,117 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:01:01,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:02,117 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:01:03,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:05,591 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:01:06,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:09,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:11,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:14,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:16,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:19,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:21,910 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:24,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:26,911 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:29,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:31,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:34,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:35,592 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:01:36,913 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:40,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:41,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:45,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:46,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:50,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:51,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:54,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:54,891 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:54,891 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:54,891 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:01:55,136 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:01:55,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:56,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:59,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:59,903 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:59,903 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:59,904 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:00,138 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:02:01,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:01,923 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:02,139 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:02:05,592 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:02:06,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:06,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:11,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:11,926 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:16,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:16,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:21,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:21,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:26,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:26,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:31,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:31,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:35,594 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:02:36,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:37,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:41,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:42,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:46,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:47,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:51,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:52,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:53,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:02:53,403 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:02:53,403 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:02:53,403 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:54,158 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:02:56,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:58,405 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:58,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:02:58,412 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:02:58,412 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:02:58,413 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:59,160 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:00,160 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:03:01,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:03,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:05,600 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:03:06,937 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:09,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:11,938 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:14,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:16,939 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:19,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:21,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:24,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:26,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:29,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:31,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:34,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:35,596 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:03:36,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:40,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:41,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:45,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:46,945 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:50,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:51,946 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:52,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:03:52,428 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:03:52,428 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:03:52,430 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:03:53,181 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:56,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:56,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:57,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:03:57,429 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:03:57,429 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:03:57,430 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:03:58,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:59,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:04:01,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:01,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:05,597 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:04:06,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:07,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:11,950 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:12,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:16,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:17,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:21,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:22,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:26,953 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:27,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:31,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:32,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:35,599 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:04:36,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:38,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:41,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:43,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:46,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:48,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:51,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:52,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:04:52,186 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:04:52,186 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:04:52,187 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:04:52,204 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:04:54,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:56,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:57,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:04:57,172 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:04:57,172 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:04:57,172 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:04:57,206 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:04:59,206 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:04:59,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:01,959 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:04,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:05,598 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:05:06,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:09,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:11,961 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:14,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:16,962 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:19,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:21,963 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:24,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:26,964 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:29,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:31,965 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:34,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:35,599 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:05:36,966 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:40,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:41,967 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:45,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:46,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:50,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:51,836 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:05:51,837 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:05:51,837 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:05:51,838 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:05:51,969 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:52,226 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:05:55,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:56,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:05:56,849 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:05:56,849 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:05:56,850 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:05:56,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:57,228 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:05:59,229 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:06:01,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:01,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:05,600 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:06:06,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:06,972 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:11,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:11,973 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:16,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:16,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:21,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:21,975 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:26,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:26,976 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:31,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:31,977 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:35,602 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:06:36,978 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:37,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:41,979 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:42,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:46,980 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:47,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:50,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:06:50,685 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:06:50,686 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:06:50,686 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:06:51,249 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:06:51,981 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:52,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:55,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:06:55,686 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:06:55,687 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:06:55,687 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:06:56,251 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:06:56,982 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:57,251 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:06:57,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:01,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:02,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:05,603 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:07:06,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:08,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:11,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:13,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:16,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:18,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:21,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:23,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:26,988 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:28,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:31,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:33,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:35,604 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:07:36,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:39,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:41,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:44,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:46,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:49,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:49,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:07:49,986 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:07:49,986 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:07:49,987 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:07:50,271 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:07:51,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:54,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:07:54,986 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:07:54,986 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:07:54,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:54,987 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:07:55,273 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:07:57,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:57,274 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:08:00,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:02,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:05,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:05,604 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:08:07,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:10,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:12,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:15,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:17,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:20,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:22,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:25,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:27,006 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:30,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:32,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:35,606 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:08:36,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:37,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:41,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:42,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:46,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:47,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:48,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:08:48,773 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:08:48,773 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:08:48,774 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:08:49,293 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:08:51,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:52,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:53,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:08:53,770 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:08:53,771 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:08:53,771 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:08:54,295 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:08:55,295 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:08:57,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:57,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:02,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:02,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:05,607 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:09:07,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:07,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:12,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:12,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:17,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:17,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:22,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:22,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:27,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:27,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:32,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:32,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:35,608 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:09:37,019 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:38,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:42,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:43,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:47,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:47,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:09:47,318 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:09:47,318 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:09:47,319 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:09:48,316 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:09:49,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:52,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:52,322 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:09:52,323 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:09:52,323 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:09:52,324 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:09:53,317 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:09:54,998 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:55,318 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:09:57,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:59,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:02,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:05,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:05,609 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:10:07,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:10,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:12,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:15,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:17,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:20,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:22,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:25,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:27,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:30,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:32,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:35,610 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:10:36,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:37,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:41,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:42,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:46,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:10:46,057 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:10:46,057 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:10:46,058 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:10:46,337 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:10:47,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:47,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:51,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:10:51,083 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:10:51,083 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:10:51,084 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:10:51,339 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:10:52,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:52,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:53,340 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:10:57,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:57,318 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:02,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:02,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:05,611 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:11:07,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:07,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:12,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:12,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:17,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:17,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:22,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:22,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:27,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:27,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:32,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:32,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:35,611 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:11:37,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:38,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:42,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:43,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:45,626 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:11:45,627 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:11:45,627 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:11:45,628 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:11:46,359 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:11:47,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:48,629 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:50,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:11:50,610 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:11:50,611 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:11:50,611 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:11:51,361 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:11:52,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:53,362 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:11:53,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:57,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:58,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:02,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:03,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:05,612 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:12:07,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:09,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:12,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:14,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:17,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:19,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:22,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:24,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:27,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:29,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:32,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:34,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:35,613 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:12:37,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:40,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:42,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:44,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:12:44,950 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:12:44,951 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:12:44,951 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:12:45,382 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:12:45,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:47,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:49,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:12:49,959 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:12:49,959 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:12:49,959 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:12:50,384 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:12:51,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:51,384 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:12:52,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:56,289 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:57,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:01,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:02,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:05,614 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:13:06,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:07,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:11,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:12,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:16,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:17,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:21,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:22,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:26,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:27,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:31,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:32,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:35,615 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:13:37,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:37,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:42,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:42,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:43,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:13:43,954 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:13:43,954 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:13:43,955 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:13:44,405 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:13:47,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:47,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:48,966 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:13:48,967 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:13:48,967 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:13:48,968 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:13:49,407 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:13:51,408 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:13:52,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:53,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:57,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:58,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:02,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:03,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:05,616 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:14:07,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:08,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:12,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:13,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:17,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:18,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:22,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:23,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:27,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:28,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:32,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:33,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:35,617 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:14:37,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:39,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:42,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:42,450 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:14:42,451 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:14:42,451 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:14:42,452 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:14:43,429 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:14:45,453 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:47,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:47,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:14:47,456 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:14:47,456 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:14:47,456 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:14:48,431 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:14:49,431 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:14:50,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:52,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:55,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:57,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:00,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:02,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:05,618 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:15:06,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:07,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:11,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:12,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:16,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:17,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:21,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:22,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:26,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:27,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:31,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:32,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:35,618 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:15:37,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:37,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:41,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:15:41,076 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:15:41,077 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:15:41,077 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:15:41,450 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:15:42,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:43,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:46,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:15:46,069 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:15:46,070 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:15:46,070 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:15:46,451 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:15:47,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:47,452 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:15:48,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:52,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:53,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:57,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:58,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:02,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:03,457 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:05,619 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:16:07,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:08,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:12,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:13,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:17,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:18,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:22,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:23,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:27,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:28,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:32,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:33,625 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:35,620 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:16:37,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:39,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:40,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:16:40,291 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:16:40,292 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:16:40,292 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:16:40,471 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:16:42,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:45,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:45,297 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:16:45,298 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:16:45,298 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:16:45,299 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:16:45,473 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:16:47,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:47,474 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:16:50,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:52,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:55,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:57,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:00,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:02,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:05,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:05,621 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:07,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:10,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:12,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:15,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:16,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:16,748 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:16,748 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:16,748 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:17,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:17,486 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:17:20,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:21,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:21,776 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:21,777 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:21,777 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:22,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:22,044 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:22,046 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:22,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:22,047 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:22,047 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:22,048 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:22,048 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:22,048 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:22,049 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:22,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:17:22,060 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:17:22,060 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:17:22,060 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:17:22,061 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:22,061 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:17:22,062 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:17:22,062 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:17:22,063 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:17:22,064 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:17:22,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:22,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:22,242 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:22,242 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:22,242 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:22,244 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:22,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:17:22,255 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:17:22,255 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:17:22,255 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:17:22,255 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:17:22,256 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:22,257 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:17:22,259 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:17:22,259 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:17:22,260 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:17:22,488 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:17:23,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:23,797 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:23,797 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:23,798 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:23,799 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:23,809 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:25,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:27,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:30,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:32,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:35,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:37,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:40,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:42,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:42,496 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 14:17:47,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:47,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:52,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:52,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:57,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:57,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:02,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:02,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:07,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:07,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:12,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:12,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:17,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:17,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:22,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:22,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:27,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:27,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:32,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:32,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:37,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:37,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:42,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:42,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:47,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:47,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:52,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:52,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:57,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:57,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:02,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:02,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:07,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:07,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:12,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:12,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:17,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:17,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:22,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:22,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:23,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:19:23,884 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:19:23,884 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:19:23,884 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:19:23,885 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:19:23,886 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:19:23,886 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:19:23,887 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:19:23,890 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:19:25,536 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:27,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:27,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:27,537 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:32,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:32,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:37,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:37,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:37,541 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:39,541 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:41,542 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:42,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:42,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:47,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:47,424 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:48,751 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:19:48,751 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:19:48,751 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:19:48,751 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:19:48,752 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:19:48,752 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:19:48,753 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:19:48,754 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:19:48,766 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:19:48,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:19:48,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:19:48,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:19:48,767 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:19:48,767 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:19:48,767 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:19:48,768 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:19:48,769 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:19:48,769 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:19:48,770 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:19:48,770 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:19:48,771 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:19:48,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:19:48,796 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:19:48,796 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:19:48,796 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:19:48,798 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:19:48,812 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:19:52,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:52,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:57,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:57,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:02,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:02,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:07,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:07,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:12,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:12,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:17,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:17,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:22,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:22,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:27,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:27,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:32,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:32,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:37,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:37,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:42,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:42,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:47,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:47,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:52,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:52,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:57,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:57,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:20:57,686 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:20:57,687 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:20:57,687 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:20:57,688 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:20:57,688 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:20:57,689 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:20:57,690 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:20:57,691 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:20:57,825 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:21:02,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:21:02,826 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report