diff --git "a/wandb/run-20240308_121134-5y42ufcl/logs/debug-internal.log" "b/wandb/run-20240308_121134-5y42ufcl/logs/debug-internal.log" --- "a/wandb/run-20240308_121134-5y42ufcl/logs/debug-internal.log" +++ "b/wandb/run-20240308_121134-5y42ufcl/logs/debug-internal.log" @@ -552,3 +552,1454 @@ 2024-03-08 12:28:40,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status 2024-03-08 12:28:40,216 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status 2024-03-08 12:28:40,429 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:28:45,430 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:28:47,333 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:28:47,333 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:28:47,333 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:28:47,334 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:28:47,334 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:28:47,334 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:28:47,334 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:28:47,335 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:28:47,341 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:28:47,341 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:28:47,341 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:28:47,341 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:28:51,342 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:28:55,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:28:55,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:28:55,216 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:28:56,414 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:01,415 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:06,417 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:10,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:29:10,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:29:10,216 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:29:11,424 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:16,425 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:16,774 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:29:16,774 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:29:16,774 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:29:16,774 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:29:16,775 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:29:16,775 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:29:16,776 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:29:16,776 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:29:16,778 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:29:21,426 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:25,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:29:25,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:29:25,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:29:27,423 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:32,424 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:37,425 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:40,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:29:40,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:29:40,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:29:42,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:47,461 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:52,462 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:29:55,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:29:55,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:29:55,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:29:58,438 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:03,439 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:08,440 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:10,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:30:10,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:30:10,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:30:13,467 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:16,779 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:30:16,780 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:30:18,781 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:23,782 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:25,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:30:25,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:30:25,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:30:29,469 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:34,470 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:39,470 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:40,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:30:40,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:30:40,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:30:44,490 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:46,788 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:30:49,789 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:54,790 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:30:55,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:30:55,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:30:55,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:31:00,439 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:05,440 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:10,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:31:10,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:31:10,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:31:11,435 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:16,437 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:16,782 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:31:18,083 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:31:18,083 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:31:18,083 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:31:18,084 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:31:18,084 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:31:18,084 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:31:18,084 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:31:18,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:31:18,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:31:18,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:31:18,092 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:31:22,093 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:25,216 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:31:25,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:31:25,217 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:31:27,493 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:32,494 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:37,495 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:40,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:31:40,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:31:40,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:31:43,041 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:31:43,042 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:31:43,042 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:31:43,042 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:31:43,043 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:31:43,043 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:31:43,044 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:31:43,044 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:31:43,044 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:31:43,045 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:31:43,046 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:31:43,046 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:31:43,046 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:31:43,046 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:31:43,046 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:31:43,048 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:31:43,056 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:31:43,057 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:31:43,057 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:31:43,057 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:48,059 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:48,555 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:31:48,556 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:31:48,556 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:31:48,556 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:31:48,557 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:31:48,558 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:31:48,559 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:31:48,559 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:31:48,560 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:31:49,034 INFO Thread-12 :10843 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:31:51,035 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:31:53,562 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:31:55,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:31:55,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:31:55,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:31:59,464 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:03,055 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:32:03,055 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:32:03,055 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:32:03,056 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:32:03,056 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:32:03,056 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:32:03,057 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:32:03,057 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:32:03,064 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:32:03,065 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:32:03,065 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:32:03,065 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:32:05,040 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:32:05,066 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:10,067 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:10,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:32:10,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:32:10,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:32:15,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:20,500 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:25,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:32:25,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:32:25,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:32:25,800 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:30,801 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:35,802 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:40,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:32:40,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:32:40,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:32:41,449 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:46,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:51,452 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:32:55,217 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:32:55,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:32:55,218 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:32:56,503 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:01,505 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:06,506 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:10,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:33:10,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:33:10,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:33:12,426 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:14,681 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:33:14,681 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:33:14,681 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:33:14,681 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:33:14,682 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:33:14,683 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:33:14,683 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:33:14,683 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:33:14,684 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:33:14,685 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:33:14,686 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:33:14,686 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:33:14,687 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:33:14,687 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:33:14,687 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:33:14,688 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:33:14,689 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:33:14,700 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:33:14,700 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:33:14,700 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:33:14,701 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:33:14,701 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:33:14,701 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:33:14,701 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:33:14,702 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:33:14,703 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:33:14,703 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:33:14,705 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:33:14,706 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:33:14,707 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:33:17,067 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:33:17,704 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:22,705 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:25,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:33:25,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:33:25,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:33:26,696 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:33:26,696 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:33:26,696 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:33:26,697 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:33:26,697 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:33:26,698 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:33:26,698 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:33:26,698 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:33:26,706 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:33:26,706 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:33:26,706 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:33:26,707 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:33:27,707 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:29,071 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:33:32,708 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:37,710 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:40,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:33:40,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:33:40,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:33:43,477 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:48,478 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:53,479 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:33:55,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:33:55,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:33:55,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:33:58,485 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:03,486 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:08,487 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:10,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:34:10,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:34:10,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:34:14,427 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:19,429 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:24,430 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:25,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:34:25,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:34:25,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:34:29,492 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:34,493 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:39,494 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:40,218 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:34:40,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:34:40,219 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:34:44,507 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:45,101 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:34:45,102 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:34:45,102 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:34:45,102 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:34:45,102 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:34:45,103 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:34:45,103 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:34:45,103 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:34:45,104 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:34:45,104 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:34:45,105 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:34:45,105 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:34:45,105 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:34:45,105 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:34:45,106 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:34:45,108 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:34:45,116 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:34:45,117 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:34:45,117 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:34:45,751 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:34:45,751 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:34:45,751 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:34:45,751 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:34:45,753 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:34:45,753 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:34:45,754 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:34:45,755 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:34:45,756 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:34:47,100 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:34:49,755 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:54,756 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:34:55,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:34:55,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:34:55,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:34:57,676 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:34:57,676 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:34:57,676 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:34:57,677 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:34:57,677 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:34:57,677 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:34:57,678 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:34:57,678 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:34:57,685 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:34:57,685 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:34:57,686 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:34:57,686 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:34:59,105 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:35:00,687 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:05,688 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:10,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:35:10,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:35:10,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:35:11,494 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:16,495 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:21,495 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:25,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:35:25,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:35:25,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:35:27,476 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:32,477 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:37,177 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:35:37,178 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:35:37,178 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:35:37,178 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:35:37,178 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:35:37,179 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:35:37,180 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:35:37,180 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:35:37,180 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:35:37,180 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:35:37,180 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:35:37,180 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:35:37,180 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:35:37,181 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:35:37,182 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:35:37,193 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:35:37,194 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:35:37,857 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:35:37,858 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:35:37,858 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:35:37,858 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:35:37,858 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:35:37,859 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:35:37,860 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:35:37,861 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:35:37,861 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:37,862 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:35:39,121 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:35:40,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:35:40,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:35:40,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:35:43,477 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:48,478 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:53,480 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:35:55,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:35:55,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:35:55,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:35:59,455 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:02,355 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:36:02,355 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:36:02,355 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:36:02,356 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:36:02,356 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:36:02,356 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:36:02,356 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:36:02,357 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:36:02,365 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:36:02,365 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:36:02,365 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:36:02,365 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:36:03,129 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:36:05,367 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:10,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:36:10,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:36:10,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:36:10,393 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:15,395 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:20,396 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:25,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:36:25,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:36:25,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:36:25,457 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:30,458 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:35,236 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:36:35,237 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:36:35,237 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:36:35,237 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:36:35,237 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:36:35,242 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:36:35,246 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:36:35,246 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:36:35,246 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:36:35,247 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:36:35,247 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:36:35,247 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:36:35,247 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:36:35,247 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:36:35,248 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:36:35,248 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:36:35,249 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:36:36,250 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:40,219 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:36:40,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:36:40,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:36:41,510 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:44,517 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:36:44,517 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:36:44,517 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:36:44,517 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:36:44,518 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:36:44,519 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:36:44,520 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:36:44,521 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:36:44,522 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:36:44,527 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:36:44,527 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:36:44,527 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:36:44,528 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:36:44,528 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:36:44,528 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:36:44,528 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:36:44,528 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:36:44,538 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:36:44,539 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:36:44,539 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:36:44,539 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:36:46,540 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:51,542 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:36:55,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:36:55,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:36:55,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:36:57,467 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:02,468 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:07,469 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:10,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:37:10,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:37:10,220 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:37:13,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:18,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:23,453 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:25,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:37:25,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:37:25,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:37:29,429 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:32,118 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:37:32,118 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:37:32,118 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:37:32,118 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:37:32,119 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:37:32,121 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:37:32,122 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:37:32,123 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:37:32,124 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:37:33,164 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:37:35,122 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:40,124 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:40,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:37:40,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:37:40,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:37:45,476 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:50,477 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:55,220 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:37:55,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:37:55,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:37:55,848 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:37:55,849 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:37:55,849 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:37:55,849 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:37:55,849 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:37:55,849 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:37:55,849 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:37:55,850 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:37:55,850 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:37:55,857 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:37:55,857 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:37:55,857 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:37:55,858 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:37:57,174 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:38:00,859 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:05,860 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:10,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:38:10,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:38:10,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:38:11,509 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:16,510 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:18,923 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:38:18,923 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:38:18,923 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:38:18,923 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:38:18,924 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:38:18,924 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:38:18,925 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:38:18,925 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:38:18,926 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:38:18,926 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:38:18,927 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:38:18,927 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:38:18,927 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:38:18,927 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:38:18,928 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:38:18,929 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:38:18,938 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:38:18,938 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:38:18,939 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:38:21,940 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:25,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:38:25,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:38:25,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:38:26,075 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:38:26,075 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:38:26,076 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:38:26,076 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:38:26,076 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:38:26,077 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:38:26,077 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:38:26,078 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:38:26,078 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:38:26,079 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:38:26,080 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:38:26,081 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:38:26,081 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:38:26,081 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:38:26,081 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:38:26,082 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:38:26,083 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:38:26,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:38:26,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:38:26,091 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:38:26,091 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:38:26,725 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:38:26,726 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:38:26,726 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:38:26,726 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:38:26,727 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:38:26,728 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:38:26,728 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:38:26,729 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:38:26,730 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:38:27,185 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:38:27,727 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:32,729 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:37,730 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:40,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:38:40,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:38:40,221 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:38:43,401 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:48,402 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:50,077 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:38:50,077 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:38:50,077 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:38:50,077 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:38:50,078 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:38:50,078 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:38:50,078 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:38:50,078 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:38:50,085 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:38:50,085 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:38:50,085 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:38:50,085 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:38:51,193 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:38:54,087 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:38:55,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:38:55,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:38:55,222 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:38:59,457 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:04,459 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:09,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:10,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:39:10,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:39:10,222 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:39:14,487 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:19,488 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:24,489 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:25,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:39:25,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:39:25,222 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:39:30,434 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:35,435 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:40,221 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:39:40,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:39:40,222 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:39:41,407 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:46,408 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:51,410 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:39:55,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:39:55,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:39:55,222 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:39:56,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:01,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:06,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:10,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:40:10,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:40:10,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:40:12,411 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:17,412 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:22,414 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:25,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:40:25,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:40:25,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:40:27,435 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:32,437 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:37,438 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:40,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:40:40,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:40:40,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:40:42,445 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:47,447 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:52,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:40:55,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:40:55,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:40:55,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:40:57,506 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:02,507 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:07,509 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:10,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:41:10,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:41:10,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:41:13,490 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:18,491 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:23,492 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:25,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:41:25,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:41:25,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:41:29,462 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:34,464 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:39,465 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:40,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:41:40,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:41:40,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:41:45,434 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:50,435 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:41:55,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:41:55,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:41:55,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:41:55,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:00,461 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:05,462 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:10,222 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:42:10,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:42:10,223 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:42:10,502 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:15,504 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:20,505 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:25,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:42:25,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:42:25,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:42:26,463 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:31,464 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:36,465 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:40,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:42:40,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:42:40,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:42:42,459 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:47,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:52,462 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:42:55,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:42:55,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:42:55,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:42:58,458 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:03,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:08,461 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:10,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:43:10,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:43:10,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:43:14,458 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:19,459 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:24,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:25,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:43:25,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:43:25,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:43:30,415 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:35,416 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:40,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:43:40,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:43:40,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:43:40,430 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:45,431 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:50,432 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:43:55,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:43:55,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:43:55,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:43:55,478 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:00,479 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:05,481 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:10,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:44:10,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:44:10,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:44:10,503 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:15,505 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:20,506 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:25,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:44:25,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:44:25,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:44:26,497 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:31,498 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:36,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:40,223 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:44:40,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:44:40,224 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:44:42,468 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:47,470 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:52,471 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:44:55,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:44:55,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:44:55,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:44:58,420 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:03,422 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:08,423 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:10,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:45:10,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:45:10,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:45:14,406 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:19,407 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:24,409 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:25,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:45:25,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:45:25,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:45:29,473 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:34,475 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:39,476 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:40,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:45:40,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:45:40,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:45:45,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:50,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:45:55,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:45:55,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:45:55,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:45:55,452 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:00,453 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:05,454 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:10,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:46:10,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:46:10,225 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:46:10,496 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:15,497 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:20,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:25,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:46:25,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:46:25,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:46:26,481 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:31,482 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:36,484 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:40,224 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:46:40,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:46:40,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:46:42,478 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:47,480 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:52,481 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:46:53,714 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:46:53,714 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:46:53,714 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:46:53,714 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:46:53,715 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:46:53,716 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:46:53,716 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:46:53,717 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:46:53,718 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:46:55,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:46:55,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:46:55,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:46:57,481 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:02,482 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:03,699 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:47:03,699 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:47:03,699 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:47:03,700 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:47:03,700 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:47:03,701 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:47:03,701 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:47:03,701 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:47:03,711 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:47:03,711 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:47:03,711 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:47:03,712 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:47:07,713 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:10,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:47:10,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:47:10,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:47:13,466 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:13,771 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:47:13,771 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:47:13,771 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:47:13,771 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:47:13,772 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:47:13,773 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:47:13,773 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:47:13,774 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:47:13,774 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:47:13,775 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:47:13,775 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:47:13,776 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:47:13,776 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:47:13,776 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:47:13,776 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:47:13,778 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:47:13,779 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:47:13,786 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:47:13,786 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:47:13,786 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:47:13,787 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:47:18,788 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:20,508 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:47:20,508 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:47:20,508 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:47:20,508 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:47:20,510 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:47:20,510 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:47:20,512 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:47:20,512 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:47:20,513 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:47:20,618 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:47:20,618 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:47:20,618 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:47:20,619 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:47:20,619 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:47:20,619 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:47:20,620 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:47:20,621 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:47:20,629 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:47:20,629 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:47:20,630 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:47:20,630 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:47:24,631 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:25,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:47:25,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:47:25,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:47:30,418 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:35,419 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:40,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:47:40,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:47:40,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:47:40,446 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:45,447 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:50,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:47:55,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:47:55,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:47:55,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:47:55,500 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:00,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:05,503 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:10,225 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:48:10,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:48:10,226 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:48:10,506 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:15,508 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:20,509 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:25,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:48:25,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:48:25,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:48:26,419 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:31,420 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:36,421 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:40,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:48:40,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:48:40,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:48:41,442 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:46,443 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:51,445 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:48:55,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:48:55,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:48:55,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:48:56,510 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:01,511 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:06,512 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:10,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:49:10,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:49:10,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:49:12,444 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:17,445 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:22,447 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:25,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:49:25,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:49:25,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:49:27,498 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:32,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:37,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:40,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:49:40,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:49:40,227 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:49:43,418 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:48,419 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:53,420 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:49:55,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:49:55,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:49:55,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:49:58,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:03,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:08,452 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:10,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:50:10,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:50:10,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:50:14,446 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:19,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:24,449 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:25,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:50:25,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:50:25,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:50:29,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:34,502 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:39,503 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:40,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:50:40,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:50:40,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:50:45,452 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:50,453 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:50:55,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:50:55,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:50:55,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:50:55,486 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:00,487 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:05,488 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:10,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:51:10,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:51:10,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:51:11,453 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:16,454 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:21,456 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:25,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:51:25,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:51:25,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:51:26,498 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:31,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:36,500 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:40,227 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:51:40,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:51:40,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:51:42,416 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:47,417 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:52,419 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:51:55,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:51:55,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:51:55,228 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:51:57,490 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:02,491 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:07,493 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:10,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:52:10,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:52:10,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:52:13,402 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:18,403 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:23,405 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:25,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:52:25,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:52:25,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:52:28,482 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:33,483 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:38,484 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:40,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:52:40,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:52:40,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:52:44,443 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:49,444 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:54,445 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:52:55,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:52:55,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:52:55,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:52:59,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:04,502 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:09,503 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:10,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:53:10,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:53:10,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:53:15,464 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:20,465 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:25,228 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:53:25,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:53:25,229 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:53:26,460 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:31,462 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:36,463 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:40,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:53:40,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:53:40,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:53:42,446 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:47,447 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:52,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:53:55,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:53:55,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:53:55,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:53:57,470 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:02,471 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:07,472 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:10,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:54:10,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:54:10,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:54:13,415 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:18,416 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:23,418 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:25,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:54:25,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:54:25,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:54:28,513 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:29,824 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:54:29,824 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:54:29,825 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:54:29,825 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:54:29,825 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:54:29,826 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:54:29,827 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:54:29,828 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:54:29,828 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:54:33,514 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:38,515 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:40,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:54:40,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:54:40,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:54:44,493 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:49,406 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:54:49,406 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:54:49,407 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:54:49,407 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:54:49,407 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:54:49,407 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:54:49,408 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:54:49,408 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:54:49,415 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:54:49,415 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:54:49,415 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:54:49,416 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:54:50,416 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:55,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:54:55,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:54:55,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:54:55,442 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:54:59,067 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:54:59,067 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:54:59,067 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:54:59,067 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:54:59,068 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:54:59,068 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:54:59,069 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:54:59,070 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:54:59,070 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:54:59,107 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:54:59,107 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:54:59,107 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:54:59,107 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:54:59,108 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:54:59,108 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:54:59,108 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:54:59,108 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:54:59,115 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:54:59,115 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:54:59,115 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:54:59,115 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:55:01,116 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:06,117 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:06,447 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:55:06,447 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:55:06,447 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:55:06,448 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:55:06,448 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:55:06,448 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:55:06,449 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:55:06,450 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:55:06,451 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:55:06,570 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:55:06,570 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:55:06,570 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:55:06,571 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:55:06,571 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:55:06,571 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:55:06,571 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:55:06,572 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:55:06,579 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:55:06,579 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:55:06,579 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:55:06,580 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:55:10,229 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:55:10,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:55:10,230 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:55:11,448 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:16,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:21,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:25,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:55:25,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:55:25,231 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:55:27,414 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:32,415 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:37,416 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:40,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:55:40,230 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:55:40,231 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:55:42,508 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:47,510 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:52,511 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:55:55,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:55:55,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:55:55,231 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:55:58,449 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:03,450 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:08,451 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:10,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:56:10,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:56:10,231 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:56:14,429 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:19,430 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:24,431 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:25,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:56:25,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:56:25,231 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:56:29,481 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:34,482 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:39,484 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:40,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:56:40,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:56:40,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:56:44,499 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:49,500 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:54,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:56:55,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:56:55,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:56:55,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:56:55,290 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:56:55,290 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:56:55,290 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:56:55,290 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:56:55,291 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:56:55,291 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:56:55,292 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:56:55,293 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:56:55,294 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:56:56,113 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:56:56,113 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:56:56,113 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:56:56,113 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:56:56,113 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:56:56,113 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:56:56,114 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:56:56,115 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:56:56,122 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:56:56,122 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:56:56,122 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:56:56,123 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:56:56,649 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:56:56,649 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:56:56,650 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:56:56,650 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:56:56,650 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:56:56,651 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:56:56,652 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:56:56,653 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:56:56,653 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:57:00,124 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:05,125 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:10,126 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:10,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:57:10,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:57:10,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:57:15,482 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:20,483 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:25,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:57:25,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:57:25,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:57:25,522 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:30,523 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:35,524 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:40,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:57:40,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:57:40,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:57:41,501 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:46,502 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:51,502 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:57:55,231 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:57:55,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:57:55,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:57:56,654 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:57:56,655 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:57:56,656 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:01,657 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:06,658 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:10,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:58:10,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:58:10,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:58:12,526 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:17,527 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:22,528 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:25,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:58:25,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:58:25,232 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:58:26,656 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:27,657 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:32,658 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:37,659 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:40,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:58:40,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:58:40,233 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:58:43,488 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:47,380 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:47,380 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:47,380 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:47,380 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:47,380 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:47,381 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:47,381 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:47,389 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:47,389 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:47,389 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:47,390 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:47,420 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:47,420 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:47,420 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:47,420 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:47,421 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:47,421 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:47,422 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:47,422 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:47,423 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:47,897 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:47,897 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:47,898 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:47,898 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:47,898 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:47,898 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:47,899 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:47,899 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:47,905 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:47,905 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:47,906 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:47,906 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:47,908 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:47,908 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:47,908 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:47,909 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:47,909 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:47,909 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:47,909 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:47,910 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:47,910 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:48,823 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:48,823 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:48,823 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:48,824 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:48,824 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:48,824 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:48,824 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:48,825 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:48,833 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:48,833 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:48,833 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:48,834 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:48,835 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:50,181 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:50,181 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:50,182 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:50,182 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:50,182 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:50,183 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:50,183 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:50,184 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:50,184 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:50,186 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:50,186 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:50,186 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:50,187 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:50,187 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:50,187 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:50,188 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:50,188 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:50,199 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:50,199 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:50,199 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:50,199 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:50,199 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:50,199 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:50,199 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:50,200 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:50,200 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:50,201 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:50,201 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:50,202 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:50,203 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:50,573 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:50,573 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:50,573 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:50,574 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:50,574 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:50,575 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:50,575 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:50,575 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:50,582 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:50,583 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:50,583 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:50,583 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:50,583 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:50,583 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:50,583 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:50,584 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:50,584 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:50,585 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:50,585 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:50,586 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:50,587 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:50,588 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:50,588 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:50,588 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:50,589 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:50,589 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:50,589 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:50,589 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:50,590 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:50,599 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:50,599 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:50,600 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:50,600 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: resume +2024-03-08 12:58:50,600 INFO HandlerThread:10843 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-08 12:58:50,600 INFO HandlerThread:10843 [system_monitor.py:start():194] Starting system monitor +2024-03-08 12:58:50,600 INFO SystemMonitor:10843 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-08 12:58:50,601 INFO SystemMonitor:10843 [interfaces.py:start():190] Started cpu monitoring +2024-03-08 12:58:50,601 INFO SystemMonitor:10843 [interfaces.py:start():190] Started disk monitoring +2024-03-08 12:58:50,602 INFO SystemMonitor:10843 [interfaces.py:start():190] Started gpu monitoring +2024-03-08 12:58:50,602 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:50,602 INFO SystemMonitor:10843 [interfaces.py:start():190] Started memory monitoring +2024-03-08 12:58:50,604 INFO SystemMonitor:10843 [interfaces.py:start():190] Started network monitoring +2024-03-08 12:58:50,692 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:58:51,046 DEBUG SenderThread:10843 [sender.py:send():378] send: config +2024-03-08 12:58:51,047 DEBUG SenderThread:10843 [sender.py:send():378] send: metric +2024-03-08 12:58:51,047 DEBUG SenderThread:10843 [sender.py:send():378] send: metric +2024-03-08 12:58:51,047 WARNING SenderThread:10843 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-03-08 12:58:52,226 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: pause +2024-03-08 12:58:52,226 INFO HandlerThread:10843 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-08 12:58:52,226 INFO HandlerThread:10843 [system_monitor.py:finish():203] Stopping system monitor +2024-03-08 12:58:52,226 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined cpu monitor +2024-03-08 12:58:52,227 DEBUG SystemMonitor:10843 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-08 12:58:52,227 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined disk monitor +2024-03-08 12:58:52,227 DEBUG SystemMonitor:10843 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-08 12:58:52,228 DEBUG SystemMonitor:10843 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-08 12:58:52,237 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined gpu monitor +2024-03-08 12:58:52,237 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined memory monitor +2024-03-08 12:58:52,237 INFO HandlerThread:10843 [interfaces.py:finish():202] Joined network monitor +2024-03-08 12:58:52,238 DEBUG SenderThread:10843 [sender.py:send():378] send: stats +2024-03-08 12:58:54,239 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:58:55,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-08 12:58:55,232 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: stop_status +2024-03-08 12:58:55,233 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: stop_status +2024-03-08 12:58:59,493 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:59:00,695 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/config.yaml +2024-03-08 12:59:01,077 DEBUG SenderThread:10843 [sender.py:send():378] send: exit +2024-03-08 12:59:01,077 INFO SenderThread:10843 [sender.py:send_exit():585] handling exit code: 0 +2024-03-08 12:59:01,077 INFO SenderThread:10843 [sender.py:send_exit():587] handling runtime: 1230 +2024-03-08 12:59:01,079 INFO SenderThread:10843 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-08 12:59:01,079 INFO SenderThread:10843 [sender.py:send_exit():593] send defer +2024-03-08 12:59:01,080 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,080 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-08 12:59:01,080 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,080 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 0 +2024-03-08 12:59:01,080 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 1 +2024-03-08 12:59:01,080 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,080 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-08 12:59:01,080 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,080 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 1 +2024-03-08 12:59:01,080 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 2 +2024-03-08 12:59:01,080 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,080 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-08 12:59:01,081 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 2 +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 3 +2024-03-08 12:59:01,081 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,081 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-08 12:59:01,081 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 3 +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 4 +2024-03-08 12:59:01,081 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,081 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-08 12:59:01,081 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 4 +2024-03-08 12:59:01,081 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 5 +2024-03-08 12:59:01,082 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,082 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-08 12:59:01,082 DEBUG SenderThread:10843 [sender.py:send():378] send: summary +2024-03-08 12:59:01,082 INFO SenderThread:10843 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-08 12:59:01,083 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,083 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 5 +2024-03-08 12:59:01,083 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 6 +2024-03-08 12:59:01,083 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,083 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-08 12:59:01,083 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,083 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 6 +2024-03-08 12:59:01,083 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 7 +2024-03-08 12:59:01,083 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: status_report +2024-03-08 12:59:01,083 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:01,083 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-08 12:59:01,083 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:01,083 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 7 +2024-03-08 12:59:01,696 INFO Thread-12 :10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/wandb-summary.json +2024-03-08 12:59:02,077 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-08 12:59:03,946 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 8 +2024-03-08 12:59:03,946 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: poll_exit +2024-03-08 12:59:03,947 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:03,947 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-08 12:59:03,947 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:03,947 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 8 +2024-03-08 12:59:03,948 INFO SenderThread:10843 [job_builder.py:build():298] Attempting to build job artifact +2024-03-08 12:59:03,949 INFO SenderThread:10843 [job_builder.py:_get_source_type():439] no source found +2024-03-08 12:59:03,949 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 9 +2024-03-08 12:59:03,949 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:03,949 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-08 12:59:03,949 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:03,949 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 9 +2024-03-08 12:59:03,949 INFO SenderThread:10843 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-08 12:59:04,078 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-08 12:59:04,697 INFO SenderThread:10843 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:59:04,698 INFO SenderThread:10843 [dir_watcher.py:finish():388] scan: /content/wandb/run-20240308_121134-5y42ufcl/files +2024-03-08 12:59:04,698 INFO SenderThread:10843 [dir_watcher.py:finish():402] scan save: /content/wandb/run-20240308_121134-5y42ufcl/files/config.yaml config.yaml +2024-03-08 12:59:04,699 INFO SenderThread:10843 [dir_watcher.py:finish():402] scan save: /content/wandb/run-20240308_121134-5y42ufcl/files/wandb-summary.json wandb-summary.json +2024-03-08 12:59:04,699 INFO SenderThread:10843 [dir_watcher.py:finish():402] scan save: /content/wandb/run-20240308_121134-5y42ufcl/files/wandb-metadata.json wandb-metadata.json +2024-03-08 12:59:04,699 INFO SenderThread:10843 [dir_watcher.py:finish():402] scan save: /content/wandb/run-20240308_121134-5y42ufcl/files/requirements.txt requirements.txt +2024-03-08 12:59:04,702 INFO SenderThread:10843 [dir_watcher.py:finish():402] scan save: /content/wandb/run-20240308_121134-5y42ufcl/files/output.log output.log +2024-03-08 12:59:04,702 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 10 +2024-03-08 12:59:04,702 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: poll_exit +2024-03-08 12:59:04,703 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:04,703 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-08 12:59:04,710 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:04,710 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 10 +2024-03-08 12:59:04,710 INFO SenderThread:10843 [file_pusher.py:finish():172] shutting down file pusher +2024-03-08 12:59:05,079 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-08 12:59:05,079 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: poll_exit +2024-03-08 12:59:05,213 INFO wandb-upload_0:10843 [upload_job.py:push():131] Uploaded file /content/wandb/run-20240308_121134-5y42ufcl/files/config.yaml +2024-03-08 12:59:05,418 INFO wandb-upload_2:10843 [upload_job.py:push():131] Uploaded file /content/wandb/run-20240308_121134-5y42ufcl/files/requirements.txt +2024-03-08 12:59:05,447 INFO wandb-upload_1:10843 [upload_job.py:push():131] Uploaded file /content/wandb/run-20240308_121134-5y42ufcl/files/wandb-summary.json +2024-03-08 12:59:05,467 INFO wandb-upload_3:10843 [upload_job.py:push():131] Uploaded file /content/wandb/run-20240308_121134-5y42ufcl/files/output.log +2024-03-08 12:59:05,668 INFO Thread-11 (_thread_body):10843 [sender.py:transition_state():613] send defer: 11 +2024-03-08 12:59:05,668 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:05,668 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-08 12:59:05,669 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:05,669 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 11 +2024-03-08 12:59:05,669 INFO SenderThread:10843 [file_pusher.py:join():178] waiting for file pusher +2024-03-08 12:59:05,669 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 12 +2024-03-08 12:59:05,669 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:05,670 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-08 12:59:05,670 DEBUG SenderThread:10843 [sender.py:send_request():405] send_request: defer +2024-03-08 12:59:05,670 INFO SenderThread:10843 [sender.py:send_request_defer():609] handle sender defer: 12 +2024-03-08 12:59:05,670 INFO SenderThread:10843 [file_stream.py:finish():595] file stream finish called +2024-03-08 12:59:06,079 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-08 12:59:06,105 WARNING StreamThr :10843 [internal.py:is_dead():414] Internal process exiting, parent pid 941 disappeared +2024-03-08 12:59:06,105 ERROR StreamThr :10843 [internal.py:wandb_internal():152] Internal process shutdown. +2024-03-08 12:59:06,234 INFO SenderThread:10843 [file_stream.py:finish():599] file stream finish is done +2024-03-08 12:59:06,234 INFO SenderThread:10843 [sender.py:transition_state():613] send defer: 13 +2024-03-08 12:59:06,234 INFO SenderThread:10843 [sender.py:finish():1521] shutting down sender +2024-03-08 12:59:06,234 INFO SenderThread:10843 [file_pusher.py:finish():172] shutting down file pusher +2024-03-08 12:59:06,234 INFO SenderThread:10843 [file_pusher.py:join():178] waiting for file pusher +2024-03-08 12:59:06,234 DEBUG HandlerThread:10843 [handler.py:handle_request():146] handle_request: defer +2024-03-08 12:59:06,234 INFO HandlerThread:10843 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-08 12:59:06,234 INFO HandlerThread:10843 [handler.py:finish():869] shutting down handler +2024-03-08 12:59:06,235 INFO WriterThread:10843 [datastore.py:close():296] close: /content/wandb/run-20240308_121134-5y42ufcl/run-5y42ufcl.wandb