diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-07-20 01:15:10,180 INFO StreamThr :179 [internal.py:wandb_internal():85] W&B internal server running at pid: 179, started at: 2024-07-20 01:15:10.179274 -2024-07-20 01:15:10,184 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status -2024-07-20 01:15:10,865 INFO WriterThread:179 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240720_011510-w16tb06h/run-w16tb06h.wandb -2024-07-20 01:15:10,867 DEBUG SenderThread:179 [sender.py:send():379] send: header -2024-07-20 01:15:10,869 DEBUG SenderThread:179 [sender.py:send():379] send: run -2024-07-20 01:15:11,108 INFO SenderThread:179 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files -2024-07-20 01:15:11,109 INFO SenderThread:179 [sender.py:_start_run_threads():1188] run started: w16tb06h with start time 1721438110.180731 -2024-07-20 01:15:11,121 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: check_version -2024-07-20 01:15:11,121 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: check_version -2024-07-20 01:15:11,186 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: run_start -2024-07-20 01:15:11,198 DEBUG HandlerThread:179 [system_info.py:__init__():26] System info init -2024-07-20 01:15:11,198 DEBUG HandlerThread:179 [system_info.py:__init__():41] System info init done -2024-07-20 01:15:11,198 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:11,199 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:11,199 INFO HandlerThread:179 [system_monitor.py:probe():214] Collecting system info -2024-07-20 01:15:11,199 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:11,200 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:11,201 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:11,202 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:11,203 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:11,207 DEBUG HandlerThread:179 [system_info.py:probe():152] Probing system -2024-07-20 01:15:11,210 DEBUG HandlerThread:179 [gitlib.py:_init_repo():56] git repository is invalid -2024-07-20 01:15:11,210 DEBUG HandlerThread:179 [system_info.py:probe():200] Probing system done -2024-07-20 01:15:11,210 DEBUG HandlerThread:179 [system_monitor.py:probe():223] {'os': 'Linux-5.15.154+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-07-20T01:15:11.207840', 'startedAt': '2024-07-20T01:15:10.170069', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '3bc9163d6d69', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.134, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5777.521297454834}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357574462890625}} -2024-07-20 01:15:11,211 INFO HandlerThread:179 [system_monitor.py:probe():224] Finished collecting system info -2024-07-20 01:15:11,211 INFO HandlerThread:179 [system_monitor.py:probe():227] Publishing system info -2024-07-20 01:15:11,211 DEBUG HandlerThread:179 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment -2024-07-20 01:15:12,110 INFO Thread-12 :179 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/conda-environment.yaml -2024-07-20 01:15:26,225 ERROR HandlerThread:179 [system_info.py:_save_conda():223] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-07-20 01:43:00,925 INFO StreamThr :151 [internal.py:wandb_internal():85] W&B internal server running at pid: 151, started at: 2024-07-20 01:43:00.925196 +2024-07-20 01:43:00,927 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status +2024-07-20 01:43:01,983 INFO WriterThread:151 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240720_014300-i371q464/run-i371q464.wandb +2024-07-20 01:43:01,985 DEBUG SenderThread:151 [sender.py:send():379] send: header +2024-07-20 01:43:01,987 DEBUG SenderThread:151 [sender.py:send():379] send: run +2024-07-20 01:43:02,876 INFO SenderThread:151 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240720_014300-i371q464/files +2024-07-20 01:43:02,877 INFO SenderThread:151 [sender.py:_start_run_threads():1188] run started: i371q464 with start time 1721439780.926187 +2024-07-20 01:43:02,889 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: check_version +2024-07-20 01:43:02,889 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: check_version +2024-07-20 01:43:02,942 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: run_start +2024-07-20 01:43:02,955 DEBUG HandlerThread:151 [system_info.py:__init__():26] System info init +2024-07-20 01:43:02,955 DEBUG HandlerThread:151 [system_info.py:__init__():41] System info init done +2024-07-20 01:43:02,955 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:02,955 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:02,956 INFO HandlerThread:151 [system_monitor.py:probe():214] Collecting system info +2024-07-20 01:43:02,956 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:02,957 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:02,957 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:02,959 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:02,959 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:02,964 DEBUG HandlerThread:151 [system_info.py:probe():152] Probing system +2024-07-20 01:43:02,968 DEBUG HandlerThread:151 [gitlib.py:_init_repo():56] git repository is invalid +2024-07-20 01:43:02,968 DEBUG HandlerThread:151 [system_info.py:probe():200] Probing system done +2024-07-20 01:43:02,968 DEBUG HandlerThread:151 [system_monitor.py:probe():223] {'os': 'Linux-5.15.154+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-07-20T01:43:02.964803', 'startedAt': '2024-07-20T01:43:00.917657', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '0b51fb85f282', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.178, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.178, 'min': 0.0, 'max': 0.0}, {'current': 2000.178, 'min': 0.0, 'max': 0.0}, {'current': 2000.178, 'min': 0.0, 'max': 0.0}, {'current': 2000.178, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5777.496021270752}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357566833496094}} +2024-07-20 01:43:02,968 INFO HandlerThread:151 [system_monitor.py:probe():224] Finished collecting system info +2024-07-20 01:43:02,968 INFO HandlerThread:151 [system_monitor.py:probe():227] Publishing system info +2024-07-20 01:43:02,968 DEBUG HandlerThread:151 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-07-20 01:43:03,879 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_014300-i371q464/files/conda-environment.yaml +2024-07-20 01:43:17,982 ERROR HandlerThread:151 [system_info.py:_save_conda():223] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 216, in _save_conda subprocess.call( @@ -37,1093 +37,493 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-07-20 01:15:26,228 DEBUG HandlerThread:179 [system_info.py:_save_conda():224] Saving conda packages done -2024-07-20 01:15:26,228 INFO HandlerThread:179 [system_monitor.py:probe():229] Finished publishing system info -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:26,237 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:15:26,238 DEBUG SenderThread:179 [sender.py:send():379] send: files -2024-07-20 01:15:26,238 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now -2024-07-20 01:15:26,565 INFO wandb-upload_0:179 [upload_job.py:push():130] Uploaded file /tmp/tmp2iby7g0owandb/cuy1wkdh-wandb-metadata.json -2024-07-20 01:15:27,114 INFO Thread-12 :179 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-metadata.json -2024-07-20 01:15:27,204 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: python_packages -2024-07-20 01:15:27,205 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: python_packages -2024-07-20 01:15:27,207 DEBUG SenderThread:179 [sender.py:send():379] send: telemetry -2024-07-20 01:15:27,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:15:27,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:27,209 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:15:27,215 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:15:27,215 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:15:27,215 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:15:27,215 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:15:27,215 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:15:27,215 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:15:27,219 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:15:27,219 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:15:27,222 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:15:27,223 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:15:27,223 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:15:28,114 INFO Thread-12 :179 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/requirements.txt -2024-07-20 01:15:28,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:29,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:30,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:31,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:31,332 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:32,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:33,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:34,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:35,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:36,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:37,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:37,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:38,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:39,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:40,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:41,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:42,205 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:15:42,206 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:15:42,246 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:42,347 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:42,728 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:15:42,728 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:15:42,728 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:42,729 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:42,729 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:42,730 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:42,731 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:42,731 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:42,732 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:42,770 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:15:42,770 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:15:42,770 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:15:42,770 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:15:42,770 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:15:42,770 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:15:42,772 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:15:42,772 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:15:42,776 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:15:42,776 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:15:42,777 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:15:42,777 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:15:43,119 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml -2024-07-20 01:15:43,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:44,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:45,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:45,925 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:15:45,925 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:15:45,925 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:45,926 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:45,926 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:45,927 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:45,928 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:45,929 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:45,932 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:45,966 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:15:45,966 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:15:45,966 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:15:45,967 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:15:45,967 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:15:45,967 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:15:45,968 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:15:45,968 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:15:45,973 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:15:45,973 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:15:45,973 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:15:45,973 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:15:46,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:47,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:48,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:48,209 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:49,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:50,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:51,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:52,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:52,509 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:15:52,509 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:15:52,509 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:52,510 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:52,510 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:52,511 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:52,511 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:52,511 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:52,512 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:52,550 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:15:52,550 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:15:52,550 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:15:52,551 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:15:52,551 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:15:52,551 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:15:52,551 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:15:52,551 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:15:52,556 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:15:52,556 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:15:52,557 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:15:52,557 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:15:53,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:53,558 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:54,210 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:54,699 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:15:54,699 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:15:54,699 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:54,699 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:54,699 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:54,700 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:54,704 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:54,704 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:54,708 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:55,211 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:55,259 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:15:55,260 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:15:55,260 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:15:55,260 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:15:55,260 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:15:55,260 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:15:55,261 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:15:55,261 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:15:55,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:15:55,267 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:15:55,267 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:15:55,267 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:15:56,211 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:57,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:15:57,206 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:15:57,246 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:58,211 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:58,284 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:15:58,284 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:15:58,284 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:15:58,285 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:15:58,285 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:15:58,286 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:15:58,286 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:15:58,287 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:15:58,287 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:15:58,842 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:15:58,849 DEBUG SenderThread:179 [sender.py:send():379] send: config -2024-07-20 01:15:58,850 DEBUG SenderThread:179 [sender.py:send():379] send: telemetry -2024-07-20 01:15:58,850 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:15:58,851 DEBUG SenderThread:179 [sender.py:send():379] send: telemetry -2024-07-20 01:15:58,851 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:15:58,851 WARNING SenderThread:179 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) -2024-07-20 01:15:58,852 DEBUG SenderThread:179 [sender.py:send():379] send: telemetry -2024-07-20 01:15:59,030 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: log_artifact -2024-07-20 01:15:59,030 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: log_artifact -2024-07-20 01:15:59,125 INFO Thread-12 :179 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/output.log -2024-07-20 01:15:59,211 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:15:59,701 INFO wandb-upload_0:179 [upload_job.py:push():88] Uploaded file /tmp/tmp_nkyjmf3/model_architecture.txt -2024-07-20 01:16:00,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:00,243 INFO SenderThread:179 [sender.py:send_request_log_artifact():1518] logged artifact model-w16tb06h - {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI5MzEzNDE4MA==', 'latestArtifact': None}} -2024-07-20 01:16:01,126 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/output.log -2024-07-20 01:16:01,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:02,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:03,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:04,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:04,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:05,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:06,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:07,212 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:08,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:09,207 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:09,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:10,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:11,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:12,206 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:16:12,206 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:16:12,246 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:13,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:14,213 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:14,352 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:15,131 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml -2024-07-20 01:16:15,335 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:16,334 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:17,460 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:18,465 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:20,258 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:20,462 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:25,106 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:26,106 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:27,462 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:16:27,462 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:16:27,502 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:30,835 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:31,189 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:32,067 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:32,091 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:32,911 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:33,911 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:34,911 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:35,911 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:36,911 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:37,613 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:37,912 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:38,912 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:39,912 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:40,912 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:41,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:42,229 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:16:42,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:16:42,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:43,321 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:43,912 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:44,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:45,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:46,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:47,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:48,613 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:16:48,913 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:49,167 DEBUG SenderThread:179 [sender.py:send():379] send: telemetry -2024-07-20 01:16:49,208 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:16:49,209 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:16:49,211 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:16:49,212 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:16:49,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:50,147 INFO Thread-12 :179 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:16:50,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:51,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:52,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:53,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:54,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:55,914 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:56,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:57,229 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:16:57,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:58,287 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:16:58,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:16:59,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:00,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:01,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:02,915 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:03,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:04,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:05,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:06,513 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:17:06,514 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:17:06,514 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:06,515 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:17:06,515 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:17:06,521 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:17:06,521 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:17:06,555 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:17:06,556 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:17:06,556 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:17:06,556 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:17:06,558 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:17:06,558 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:17:06,709 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:17:06,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:07,153 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:17:07,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:08,916 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:09,917 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:10,917 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:11,711 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:11,917 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:12,229 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:17:12,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:17:12,917 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:13,917 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:14,918 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:15,918 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:16,918 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:17,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:17,918 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:18,918 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:19,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:20,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:21,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:22,297 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:22,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:23,160 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml -2024-07-20 01:17:23,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:24,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:25,919 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:26,920 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:27,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:17:27,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:17:27,920 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:28,290 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:17:28,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:28,920 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:29,920 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:30,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:31,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:32,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:33,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:33,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:34,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:35,921 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:36,922 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:37,922 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:38,292 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:38,922 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:39,922 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:40,922 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:41,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:42,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:17:42,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:17:42,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:43,323 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:43,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:44,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:45,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:46,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:47,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:48,324 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:48,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:49,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:50,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:51,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:52,924 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:53,324 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:53,925 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:54,925 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:55,925 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:56,925 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:57,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:17:57,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:17:57,926 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:58,290 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:17:58,926 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:17:59,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:17:59,926 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:00,926 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:01,926 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:02,927 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:03,927 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:04,292 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:04,927 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:05,927 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:06,927 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:07,928 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:08,928 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:09,293 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:09,928 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:10,928 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:11,928 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:12,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:18:12,230 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:18:12,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:13,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:14,357 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:14,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:15,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:16,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:17,929 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:18,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:19,358 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:19,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:20,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:21,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:22,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:23,930 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:24,359 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:24,931 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:25,931 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:26,931 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:27,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:18:27,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:18:27,931 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:28,291 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:18:28,931 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:29,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:30,292 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:30,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:31,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:32,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:33,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:34,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:35,293 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:35,932 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:36,933 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:37,933 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:38,933 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:39,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:40,294 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:40,933 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:41,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:42,230 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:18:42,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:18:42,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:43,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:44,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:45,372 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:45,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:46,934 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:47,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:48,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:49,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:50,373 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:50,374 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:18:50,374 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:18:50,374 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:18:50,375 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:18:50,376 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:18:50,377 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:18:50,384 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:18:50,385 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:18:50,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:51,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:52,935 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:53,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:54,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:55,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:18:55,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:56,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:18:57,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:18:57,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:58,936 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:18:59,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:00,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:01,315 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:01,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:02,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:03,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:04,937 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:05,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:06,316 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:06,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:07,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:08,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:09,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:10,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:11,317 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:11,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:19:12,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:19:12,938 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:13,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:14,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:15,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:16,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:17,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:17,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:18,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:19,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:20,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:21,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:22,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:22,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:23,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:24,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:25,940 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:26,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:19:27,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:19:27,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:27,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:28,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:29,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:30,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:31,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:32,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:32,941 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:33,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:34,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:35,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:36,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:37,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:37,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:38,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:39,942 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:40,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:41,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:19:42,231 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:19:42,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:43,309 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:43,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:44,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:45,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:46,943 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:47,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:48,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:48,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:49,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:50,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:51,944 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:52,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:53,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:53,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:54,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:55,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:56,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:57,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:19:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:19:57,945 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:58,358 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:19:58,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:19:59,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:00,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:01,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:02,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:03,359 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:03,946 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:04,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:05,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:06,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:07,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:08,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:08,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:09,947 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:10,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:11,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:12,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:20:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:20:12,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:13,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:14,353 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:14,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:15,948 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:16,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:17,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:18,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:19,354 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:19,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:20,949 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:21,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:22,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:23,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:24,355 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:24,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:25,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:26,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:27,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:20:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:20:27,950 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:28,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:29,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:30,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:30,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:31,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:32,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:33,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:34,951 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:35,295 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:35,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:36,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:37,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:38,952 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:39,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:40,296 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:40,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:41,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:42,231 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:20:42,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:20:42,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:43,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:44,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:45,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:45,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:46,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:47,953 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:48,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:49,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:50,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:50,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:51,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:52,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:53,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:54,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:55,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:20:55,954 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:56,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:20:57,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:20:57,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:58,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:20:59,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:00,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:01,341 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:01,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:02,955 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:03,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:04,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:05,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:06,342 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:06,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:07,956 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:08,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:21:08,291 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:21:08,291 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:21:08,291 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:21:08,291 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:21:08,292 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:21:08,293 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:21:08,294 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:21:08,295 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:21:09,123 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:09,256 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:21:09,257 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:21:09,257 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:21:09,257 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:21:09,258 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:21:09,265 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:21:09,266 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:21:09,266 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:21:09,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:10,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:11,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:12,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:21:12,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:21:12,360 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:12,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:13,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:14,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:15,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:16,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:17,361 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:17,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:18,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:19,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:20,968 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:21,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:22,362 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:22,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:23,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:24,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:25,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:26,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:27,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:21:27,232 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:21:27,404 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:27,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:28,969 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:29,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:30,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:31,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:32,405 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:32,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:33,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:34,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:35,970 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:36,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:37,406 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:37,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:38,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:39,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:40,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:41,971 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:42,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:21:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:21:42,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:43,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:43,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:44,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:45,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:46,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:47,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:48,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:48,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:49,972 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:50,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:51,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:52,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:53,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:53,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:54,973 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:55,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:56,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:57,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:21:57,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:21:57,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:58,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:21:59,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:21:59,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:00,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:01,974 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:02,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:03,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:04,290 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:04,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:05,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:06,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:07,975 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:08,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:09,291 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:09,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:10,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:11,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:22:12,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:22:12,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:13,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:14,326 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:14,976 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:15,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:16,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:17,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:18,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:19,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:19,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:20,977 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:21,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:22,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:23,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:24,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:24,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:25,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:26,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:27,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:22:27,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:22:27,978 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:28,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:29,398 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:29,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:30,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:31,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:32,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:33,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:34,399 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:34,979 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:35,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:36,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:37,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:38,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:39,400 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:39,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:40,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:41,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:42,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:22:42,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:22:42,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:43,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:44,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:45,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:45,981 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:46,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:47,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:48,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:49,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:50,311 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:50,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:51,982 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:52,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:53,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:54,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:55,312 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:22:55,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:56,983 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:57,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:22:57,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:22:57,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:58,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:22:59,714 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:22:59,715 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:22:59,715 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:22:59,715 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:22:59,716 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:22:59,717 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:22:59,720 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:22:59,721 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:22:59,756 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:22:59,756 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:22:59,756 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:22:59,757 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:22:59,757 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:22:59,758 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:22:59,763 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:22:59,764 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:22:59,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:00,463 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:23:00,463 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:23:00,463 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:23:00,463 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:23:00,464 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:23:00,465 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:23:00,470 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:23:00,503 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:23:00,503 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:23:00,503 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:23:00,503 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:23:00,504 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:23:00,504 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:23:00,509 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:23:00,510 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:23:00,510 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:23:00,511 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:00,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:01,882 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:23:01,882 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:23:01,883 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:23:01,883 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:23:01,883 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:23:01,884 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:23:01,885 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:23:01,923 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:23:01,923 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:23:01,923 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:23:01,924 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:23:01,924 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:23:01,930 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:23:01,930 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:23:01,984 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:02,980 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:23:02,980 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:23:02,980 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:23:02,980 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:23:02,981 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:23:02,982 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:23:02,983 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:23:02,984 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:23:03,020 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:23:03,020 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:23:03,020 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:23:03,020 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:23:03,021 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:23:03,021 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:23:03,026 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:23:03,027 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:23:03,027 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:03,027 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:23:03,185 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:23:03,185 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:23:03,185 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:23:03,185 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:23:03,186 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:23:03,187 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:23:03,188 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:23:03,189 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:23:03,190 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:23:03,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: pause -2024-07-20 01:23:03,327 INFO HandlerThread:179 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-07-20 01:23:03,327 INFO HandlerThread:179 [system_monitor.py:finish():203] Stopping system monitor -2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined cpu monitor -2024-07-20 01:23:03,328 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:23:03,328 INFO HandlerThread:179 [interfaces.py:finish():200] Joined disk monitor -2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-07-20 01:23:03,329 DEBUG SystemMonitor:179 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined gpu monitor -2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined memory monitor -2024-07-20 01:23:03,335 INFO HandlerThread:179 [interfaces.py:finish():200] Joined network monitor -2024-07-20 01:23:03,336 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:23:03,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:04,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:05,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:06,149 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: resume -2024-07-20 01:23:06,149 INFO HandlerThread:179 [handler.py:handle_request_resume():715] starting system metrics thread -2024-07-20 01:23:06,149 INFO HandlerThread:179 [system_monitor.py:start():194] Starting system monitor -2024-07-20 01:23:06,150 INFO SystemMonitor:179 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-07-20 01:23:06,150 INFO SystemMonitor:179 [interfaces.py:start():188] Started cpu monitoring -2024-07-20 01:23:06,151 INFO SystemMonitor:179 [interfaces.py:start():188] Started disk monitoring -2024-07-20 01:23:06,152 INFO SystemMonitor:179 [interfaces.py:start():188] Started gpu monitoring -2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started memory monitoring -2024-07-20 01:23:06,153 INFO SystemMonitor:179 [interfaces.py:start():188] Started network monitoring -2024-07-20 01:23:06,337 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:06,874 DEBUG SenderThread:179 [sender.py:send():379] send: config -2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:23:06,875 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:23:06,875 WARNING SenderThread:179 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) -2024-07-20 01:23:06,967 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: log_artifact -2024-07-20 01:23:06,967 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: log_artifact -2024-07-20 01:23:06,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:07,078 INFO SenderThread:179 [sender.py:send_request_log_artifact():1518] logged artifact model-w16tb06h - {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI5MzEzNDE4MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTAwNDIwNjIwMg==', 'versionIndex': 0}}} -2024-07-20 01:23:07,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:08,903 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: metric -2024-07-20 01:23:08,905 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:08,906 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:08,906 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:08,985 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:09,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:09,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:10,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:10,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:10,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:10,285 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:10,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:11,576 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:11,578 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:11,578 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:11,579 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:11,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:12,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:23:12,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:23:12,286 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:12,868 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:12,869 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:12,870 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:12,870 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:12,986 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:13,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:13,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:14,247 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:14,248 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:14,249 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:14,249 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:14,287 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:14,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:15,711 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:15,712 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:15,713 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:15,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:16,288 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:16,713 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:16,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:17,034 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:17,035 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:17,036 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:17,036 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:17,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:17,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:18,377 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:18,378 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:18,379 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:18,379 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:18,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:19,289 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:19,834 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:19,836 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:19,836 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:19,987 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:20,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:20,725 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:23:20,727 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:23:20,727 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:23:20,988 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:21,290 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:23:21,728 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:22,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:23,470 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:24,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:25,473 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:26,729 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:26,939 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:23:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:23:31,274 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: internal_messages -2024-07-20 01:23:32,356 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:37,072 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:23:37,364 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:38,298 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/config.yaml -2024-07-20 01:23:42,074 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:23:46,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:23:50,052 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:23:52,076 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:23:55,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:23:57,082 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:00,225 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:01,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:24:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:24:05,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:06,153 DEBUG SystemMonitor:179 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-07-20 01:24:06,154 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:24:07,084 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:11,155 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:12,085 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:16,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:16,232 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:24:16,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:24:16,568 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,568 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:16,571 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: partial_history -2024-07-20 01:24:16,572 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:16,584 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,584 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,585 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:16,585 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send():379] send: history -2024-07-20 01:24:16,586 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: summary_record -2024-07-20 01:24:16,586 INFO SenderThread:179 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end -2024-07-20 01:24:17,312 INFO Thread-12 :179 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_011510-w16tb06h/files/wandb-summary.json -2024-07-20 01:24:21,587 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:22,087 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:26,588 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:27,088 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:31,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:24:31,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:24:32,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:36,155 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:24:37,090 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:38,156 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:42,091 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:43,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:46,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:24:46,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:24:48,393 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:52,094 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:53,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:24:57,095 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:24:58,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:01,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:25:01,233 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:25:04,381 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:06,156 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:25:07,098 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:10,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:12,099 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:15,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:16,233 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:25:16,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:25:20,394 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:22,101 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:25,395 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:27,102 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:30,396 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:31,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:25:31,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:25:36,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:25:36,157 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:37,104 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:41,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:42,106 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:46,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:46,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:25:46,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:25:51,327 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:52,108 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:25:56,328 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:25:57,110 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:26:01,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:26:01,234 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:26:02,310 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:26:06,157 DEBUG SenderThread:179 [sender.py:send():379] send: stats -2024-07-20 01:26:07,112 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:26:08,158 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:26:12,113 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive -2024-07-20 01:26:13,159 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:26:16,234 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: stop_status -2024-07-20 01:26:16,235 DEBUG SenderThread:179 [sender.py:send_request():406] send_request: stop_status -2024-07-20 01:26:18,386 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: status_report -2024-07-20 01:26:22,115 DEBUG HandlerThread:179 [handler.py:handle_request():158] handle_request: keepalive +2024-07-20 01:43:17,985 DEBUG HandlerThread:151 [system_info.py:_save_conda():224] Saving conda packages done +2024-07-20 01:43:17,985 INFO HandlerThread:151 [system_monitor.py:probe():229] Finished publishing system info +2024-07-20 01:43:17,993 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:17,993 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: keepalive +2024-07-20 01:43:17,993 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:17,993 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: keepalive +2024-07-20 01:43:17,994 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:17,994 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: keepalive +2024-07-20 01:43:17,994 DEBUG SenderThread:151 [sender.py:send():379] send: files +2024-07-20 01:43:17,995 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-07-20 01:43:18,321 INFO wandb-upload_0:151 [upload_job.py:push():130] Uploaded file /tmp/tmpg7st13gowandb/5ci03tu9-wandb-metadata.json +2024-07-20 01:43:18,881 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-metadata.json +2024-07-20 01:43:18,956 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: python_packages +2024-07-20 01:43:18,956 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: python_packages +2024-07-20 01:43:18,958 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry +2024-07-20 01:43:18,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:43:18,960 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:43:18,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:18,965 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: pause +2024-07-20 01:43:18,965 INFO HandlerThread:151 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-20 01:43:18,965 INFO HandlerThread:151 [system_monitor.py:finish():203] Stopping system monitor +2024-07-20 01:43:18,966 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:43:18,966 INFO HandlerThread:151 [interfaces.py:finish():200] Joined cpu monitor +2024-07-20 01:43:18,966 DEBUG SystemMonitor:151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-20 01:43:18,967 INFO HandlerThread:151 [interfaces.py:finish():200] Joined disk monitor +2024-07-20 01:43:18,967 DEBUG SystemMonitor:151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-20 01:43:18,972 INFO HandlerThread:151 [interfaces.py:finish():200] Joined gpu monitor +2024-07-20 01:43:18,973 INFO HandlerThread:151 [interfaces.py:finish():200] Joined memory monitor +2024-07-20 01:43:18,973 INFO HandlerThread:151 [interfaces.py:finish():200] Joined network monitor +2024-07-20 01:43:19,055 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:43:19,882 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_014300-i371q464/files/requirements.txt +2024-07-20 01:43:19,957 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:20,957 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:21,055 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:21,957 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:22,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:23,957 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:24,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:25,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:26,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:26,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:27,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:28,704 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: resume +2024-07-20 01:43:28,704 INFO HandlerThread:151 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-20 01:43:28,704 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:28,705 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:28,705 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:28,706 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:28,707 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:28,709 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:28,710 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:28,745 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: pause +2024-07-20 01:43:28,745 INFO HandlerThread:151 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-20 01:43:28,745 INFO HandlerThread:151 [system_monitor.py:finish():203] Stopping system monitor +2024-07-20 01:43:28,745 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:43:28,745 DEBUG SystemMonitor:151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-20 01:43:28,745 DEBUG SystemMonitor:151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-20 01:43:28,746 INFO HandlerThread:151 [interfaces.py:finish():200] Joined cpu monitor +2024-07-20 01:43:28,746 INFO HandlerThread:151 [interfaces.py:finish():200] Joined disk monitor +2024-07-20 01:43:28,751 INFO HandlerThread:151 [interfaces.py:finish():200] Joined gpu monitor +2024-07-20 01:43:28,751 INFO HandlerThread:151 [interfaces.py:finish():200] Joined memory monitor +2024-07-20 01:43:28,751 INFO HandlerThread:151 [interfaces.py:finish():200] Joined network monitor +2024-07-20 01:43:28,751 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:43:28,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:29,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:30,222 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: resume +2024-07-20 01:43:30,222 INFO HandlerThread:151 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-20 01:43:30,223 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:30,223 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:30,223 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:30,224 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:30,225 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:30,226 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:30,226 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:30,263 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: pause +2024-07-20 01:43:30,263 INFO HandlerThread:151 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-20 01:43:30,263 INFO HandlerThread:151 [system_monitor.py:finish():203] Stopping system monitor +2024-07-20 01:43:30,263 INFO HandlerThread:151 [interfaces.py:finish():200] Joined cpu monitor +2024-07-20 01:43:30,264 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:43:30,264 DEBUG SystemMonitor:151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-20 01:43:30,264 DEBUG SystemMonitor:151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-20 01:43:30,265 INFO HandlerThread:151 [interfaces.py:finish():200] Joined disk monitor +2024-07-20 01:43:30,269 INFO HandlerThread:151 [interfaces.py:finish():200] Joined gpu monitor +2024-07-20 01:43:30,269 INFO HandlerThread:151 [interfaces.py:finish():200] Joined memory monitor +2024-07-20 01:43:30,270 INFO HandlerThread:151 [interfaces.py:finish():200] Joined network monitor +2024-07-20 01:43:30,270 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:43:30,959 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:31,959 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:32,277 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:32,886 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/config.yaml +2024-07-20 01:43:32,959 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:33,956 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:43:33,957 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:43:33,997 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:34,959 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:35,371 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: resume +2024-07-20 01:43:35,371 INFO HandlerThread:151 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-20 01:43:35,371 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:35,371 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:35,372 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:35,373 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:35,373 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:35,374 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:35,376 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:35,412 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: pause +2024-07-20 01:43:35,412 INFO HandlerThread:151 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-20 01:43:35,412 INFO HandlerThread:151 [system_monitor.py:finish():203] Stopping system monitor +2024-07-20 01:43:35,412 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:43:35,412 DEBUG SystemMonitor:151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-20 01:43:35,412 DEBUG SystemMonitor:151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-20 01:43:35,413 INFO HandlerThread:151 [interfaces.py:finish():200] Joined cpu monitor +2024-07-20 01:43:35,413 INFO HandlerThread:151 [interfaces.py:finish():200] Joined disk monitor +2024-07-20 01:43:35,418 INFO HandlerThread:151 [interfaces.py:finish():200] Joined gpu monitor +2024-07-20 01:43:35,418 INFO HandlerThread:151 [interfaces.py:finish():200] Joined memory monitor +2024-07-20 01:43:35,418 INFO HandlerThread:151 [interfaces.py:finish():200] Joined network monitor +2024-07-20 01:43:35,419 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:43:35,959 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:36,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:37,420 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:37,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:38,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:39,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:40,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:41,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:42,958 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:42,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:43,960 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:44,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:45,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:46,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:47,397 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: resume +2024-07-20 01:43:47,397 INFO HandlerThread:151 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-20 01:43:47,397 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:47,398 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:47,398 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:47,398 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:47,400 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:47,401 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:47,403 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:47,910 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: pause +2024-07-20 01:43:47,910 INFO HandlerThread:151 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-20 01:43:47,911 INFO HandlerThread:151 [system_monitor.py:finish():203] Stopping system monitor +2024-07-20 01:43:47,911 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:43:47,911 INFO HandlerThread:151 [interfaces.py:finish():200] Joined cpu monitor +2024-07-20 01:43:47,912 DEBUG SystemMonitor:151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-20 01:43:47,912 INFO HandlerThread:151 [interfaces.py:finish():200] Joined disk monitor +2024-07-20 01:43:47,912 DEBUG SystemMonitor:151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-20 01:43:47,918 INFO HandlerThread:151 [interfaces.py:finish():200] Joined gpu monitor +2024-07-20 01:43:47,918 INFO HandlerThread:151 [interfaces.py:finish():200] Joined memory monitor +2024-07-20 01:43:47,919 INFO HandlerThread:151 [interfaces.py:finish():200] Joined network monitor +2024-07-20 01:43:47,919 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:43:47,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:48,920 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:48,956 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:43:48,957 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:43:48,997 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:49,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:50,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:51,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:52,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:53,121 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: resume +2024-07-20 01:43:53,121 INFO HandlerThread:151 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-20 01:43:53,121 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor +2024-07-20 01:43:53,122 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-20 01:43:53,122 INFO SystemMonitor:151 [interfaces.py:start():188] Started cpu monitoring +2024-07-20 01:43:53,123 INFO SystemMonitor:151 [interfaces.py:start():188] Started disk monitoring +2024-07-20 01:43:53,124 INFO SystemMonitor:151 [interfaces.py:start():188] Started gpu monitoring +2024-07-20 01:43:53,125 INFO SystemMonitor:151 [interfaces.py:start():188] Started memory monitoring +2024-07-20 01:43:53,125 INFO SystemMonitor:151 [interfaces.py:start():188] Started network monitoring +2024-07-20 01:43:53,654 DEBUG SenderThread:151 [sender.py:send():379] send: config +2024-07-20 01:43:53,656 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry +2024-07-20 01:43:53,656 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:53,657 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry +2024-07-20 01:43:53,657 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:53,657 WARNING SenderThread:151 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) +2024-07-20 01:43:53,657 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry +2024-07-20 01:43:53,841 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: log_artifact +2024-07-20 01:43:53,842 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: log_artifact +2024-07-20 01:43:53,894 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_014300-i371q464/files/output.log +2024-07-20 01:43:53,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:54,494 INFO wandb-upload_0:151 [upload_job.py:push():88] Uploaded file /tmp/tmp7ma2gqyf/model_architecture.txt +2024-07-20 01:43:54,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:55,022 INFO SenderThread:151 [sender.py:send_request_log_artifact():1518] logged artifact model-i371q464 - {'id': 'QXJ0aWZhY3Q6MTAwNDI3ODQ2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI5MzE5MzI0NA==', 'latestArtifact': None}} +2024-07-20 01:43:55,022 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:43:55,894 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/output.log +2024-07-20 01:43:55,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:56,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:57,071 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send():379] send: metric +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:43:57,074 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:43:57,075 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:43:57,895 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:43:57,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:58,433 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:43:58,434 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:43:58,435 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:43:58,436 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:43:58,896 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:43:58,962 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:43:59,766 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:43:59,767 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:43:59,768 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:43:59,768 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:43:59,896 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:43:59,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:00,769 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:00,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:01,569 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:01,570 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:01,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:01,571 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:01,897 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:01,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:02,895 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:02,896 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:02,897 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:02,897 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:02,898 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:02,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:03,957 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:44:03,957 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:44:03,997 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:04,369 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:04,371 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:04,371 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:04,371 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:04,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:04,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:05,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:05,734 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:05,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:05,735 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:05,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:05,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:06,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:06,900 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/config.yaml +2024-07-20 01:44:06,963 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:07,056 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:07,058 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:07,058 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:07,058 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:07,900 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:07,964 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:08,512 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:08,513 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:08,513 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:08,515 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:08,901 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:08,964 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:09,404 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:09,405 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:09,406 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:09,406 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:09,901 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:09,964 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:11,115 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:12,115 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:12,407 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:13,240 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:14,251 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:16,090 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:17,961 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:22,331 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:44:22,331 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:44:22,371 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:23,486 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:28,488 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:28,686 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:28,998 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:29,717 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:30,717 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:31,717 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:32,717 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:33,687 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:33,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:34,717 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:35,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:36,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:37,328 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:44:37,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:44:37,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:38,688 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:38,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:39,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:40,718 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:41,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:42,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:43,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:44,688 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:44,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:45,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:46,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:47,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:48,719 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:49,688 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:49,720 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:50,056 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry +2024-07-20 01:44:50,097 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: partial_history +2024-07-20 01:44:50,098 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:50,720 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:50,918 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:51,720 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:51,964 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:51,964 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:51,965 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:51,965 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:51,967 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:51,967 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:52,017 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:52,018 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:52,021 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:52,021 DEBUG SenderThread:151 [sender.py:send():379] send: history +2024-07-20 01:44:52,021 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record +2024-07-20 01:44:52,266 INFO SenderThread:151 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-07-20 01:44:52,337 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:44:52,338 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:44:52,720 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:52,919 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/wandb-summary.json +2024-07-20 01:44:53,126 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-20 01:44:53,127 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:44:53,720 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:54,693 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:44:54,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:55,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:56,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:57,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:58,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:44:59,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:00,689 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:00,721 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:01,722 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:02,722 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:03,724 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:04,724 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:05,690 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:05,724 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:06,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:07,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:45:07,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:45:07,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:08,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:09,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:10,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:11,695 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:11,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:11,926 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240720_014300-i371q464/files/config.yaml +2024-07-20 01:45:12,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:13,725 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:14,726 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:15,726 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:16,726 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:17,690 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:17,726 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:18,726 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:19,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:20,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:21,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:22,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:45:22,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:45:22,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:23,128 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:45:23,128 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:23,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:24,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:25,727 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:26,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:27,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:28,690 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:28,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:29,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:30,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:31,728 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:32,729 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:33,690 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:33,729 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:34,729 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:35,729 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:36,729 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:37,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:45:37,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:45:37,730 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:38,690 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:38,730 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:39,730 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:40,730 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:41,730 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:42,731 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:43,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:43,731 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:44,731 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:45,731 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:46,731 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:47,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:48,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:48,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:49,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:50,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:51,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:52,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:45:52,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:45:52,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:53,129 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:45:53,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:53,732 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:54,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:55,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:56,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:57,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:58,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:45:58,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:45:59,733 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:00,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:01,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:02,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:03,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:03,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:04,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:05,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:06,734 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:07,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:46:07,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:46:07,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:08,691 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:08,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:09,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:11,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:12,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:13,692 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:13,735 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:14,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:15,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:16,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:17,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:18,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:19,692 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:19,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:20,736 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:21,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:22,329 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:46:22,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:46:22,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:23,129 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:46:23,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:24,692 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:24,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:25,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:26,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:27,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:28,737 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:29,692 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:29,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:30,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:31,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:32,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:33,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:34,692 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:34,738 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:35,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:36,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:37,330 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:46:37,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:46:37,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:38,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:39,693 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:39,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:40,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:41,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:42,740 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:43,739 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:44,740 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:45,693 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:45,740 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:46,740 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:47,740 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:48,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:49,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:50,695 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: status_report +2024-07-20 01:46:50,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:51,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:52,330 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: stop_status +2024-07-20 01:46:52,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status +2024-07-20 01:46:52,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:53,130 DEBUG SenderThread:151 [sender.py:send():379] send: stats +2024-07-20 01:46:53,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages +2024-07-20 01:46:54,741 DEBUG HandlerThread:151 [handler.py:handle_request():158] handle_request: internal_messages