diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,1620 @@ +2024-09-16 18:20:41,477 INFO StreamThr :1120898 [internal.py:wandb_internal():85] W&B internal server running at pid: 1120898, started at: 2024-09-16 18:20:41.474928 +2024-09-16 18:20:41,480 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status +2024-09-16 18:20:41,492 INFO WriterThread:1120898 [datastore.py:open_for_write():87] open: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/run-ye308qxr.wandb +2024-09-16 18:20:41,496 DEBUG SenderThread:1120898 [sender.py:send():391] send: header +2024-09-16 18:20:41,501 DEBUG SenderThread:1120898 [sender.py:send():391] send: run +2024-09-16 18:20:41,895 INFO SenderThread:1120898 [dir_watcher.py:__init__():211] watching files in: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files +2024-09-16 18:20:41,895 INFO SenderThread:1120898 [sender.py:_start_run_threads():1200] run started: ye308qxr with start time 1726507241.478331 +2024-09-16 18:20:41,911 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: run_start +2024-09-16 18:20:41,992 DEBUG HandlerThread:1120898 [system_info.py:__init__():26] System info init +2024-09-16 18:20:41,993 DEBUG HandlerThread:1120898 [system_info.py:__init__():41] System info init done +2024-09-16 18:20:41,993 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:20:41,994 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:20:41,994 INFO HandlerThread:1120898 [system_monitor.py:probe():214] Collecting system info +2024-09-16 18:20:41,995 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:20:41,999 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:20:42,001 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:20:42,004 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 18:20:42,008 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 18:20:42,035 DEBUG HandlerThread:1120898 [system_info.py:probe():152] Probing system +2024-09-16 18:20:42,039 DEBUG HandlerThread:1120898 [gitlib.py:_init_repo():56] git repository is invalid +2024-09-16 18:20:42,039 DEBUG HandlerThread:1120898 [system_info.py:probe():200] Probing system done +2024-09-16 18:20:42,039 DEBUG HandlerThread:1120898 [system_monitor.py:probe():223] {'os': 'Linux-4.18.0-513.24.1.el8_9.x86_64-x86_64-with-glibc2.28', 'python': '3.11.4', 'heartbeatAt': '2024-09-16T17:20:42.035382', 'startedAt': '2024-09-16T17:20:41.449030', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '', 'codePathLocal': None, 'host': 'codon-gpu-014.ebi.ac.uk', 'username': 'amrufai', 'executable': '/nfs/production/literature/amina-mardiyyah/envs/llm-prompt/bin/python', 'cpu_count': 48, 'cpu_count_logical': 48, 'cpu_freq': {'current': 2869.226979166667, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3251.96, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3246.317, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3260.095, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3254.547, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3242.782, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3219.073, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3248.226, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3244.463, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3242.955, 'min': 0.0, 'max': 0.0}, {'current': 803.795, 'min': 0.0, 'max': 0.0}, {'current': 3240.712, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3232.618, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 3235.352, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 47.760292053222656, 'used': 15.848091125488281}}, 'gpu': 'NVIDIA A100 80GB PCIe', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100 80GB PCIe', 'memory_total': 85899345920}], 'memory': {'total': 502.83758544921875}} +2024-09-16 18:20:42,039 INFO HandlerThread:1120898 [system_monitor.py:probe():224] Finished collecting system info +2024-09-16 18:20:42,039 INFO HandlerThread:1120898 [system_monitor.py:probe():227] Publishing system info +2024-09-16 18:20:42,043 INFO HandlerThread:1120898 [system_monitor.py:probe():229] Finished publishing system info +2024-09-16 18:20:42,237 DEBUG SenderThread:1120898 [sender.py:send():391] send: files +2024-09-16 18:20:42,238 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-metadata.json with policy now +2024-09-16 18:20:42,741 INFO wandb-upload_0:1120898 [upload_job.py:push():130] Uploaded file /tmp/tmp89w42yrbwandb/5v2rqwc0-wandb-metadata.json +2024-09-16 18:20:42,869 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: python_packages +2024-09-16 18:20:42,870 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: python_packages +2024-09-16 18:20:42,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:20:42,874 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:20:42,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:20:42,888 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 18:20:42,888 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 18:20:42,889 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 18:20:42,889 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:20:42,889 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 18:20:42,890 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 18:20:42,890 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 18:20:42,894 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 18:20:42,900 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/requirements.txt +2024-09-16 18:20:42,900 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-metadata.json +2024-09-16 18:20:42,953 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 18:20:42,954 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 18:20:42,954 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 18:20:43,033 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 18:20:43,034 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:20:47,036 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:20:52,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:20:52,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:20:57,870 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:20:57,871 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:20:58,085 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:02,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:03,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:08,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:12,870 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:21:12,871 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:21:12,911 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:14,100 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:14,929 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/config.yaml +2024-09-16 18:21:19,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:22,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:25,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:27,870 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:21:27,871 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:21:31,057 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:32,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:36,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:42,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:21:42,871 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:21:42,911 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:43,066 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:48,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:52,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:21:53,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:21:57,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:21:57,871 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:21:59,017 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:02,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:04,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:10,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:12,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:22:12,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:22:12,912 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:16,068 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:21,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:22,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:26,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:27,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:22:27,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:22:30,479 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 18:22:30,479 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 18:22:30,479 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:22:30,479 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:22:30,481 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:22:30,484 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:22:30,485 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 18:22:30,486 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 18:22:30,486 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 18:22:30,488 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 18:22:30,489 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:22:30,489 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 18:22:30,490 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:22:30,491 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 18:22:30,491 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 18:22:30,545 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 18:22:30,546 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:22:32,548 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:32,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:37,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:42,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:22:42,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:22:42,912 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:43,056 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:48,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:52,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:22:53,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:22:57,871 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:22:57,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:22:59,091 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:02,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:04,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:10,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:12,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:23:12,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:23:12,912 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:16,100 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:21,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:22,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:26,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:27,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:23:27,872 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:23:32,034 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:32,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:37,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:42,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:23:42,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:23:42,913 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:42,999 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:48,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:52,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:23:53,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:23:57,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:23:57,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:23:59,744 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:02,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:04,647 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 18:24:04,647 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 18:24:04,647 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:24:04,647 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:24:04,650 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:24:04,653 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:24:04,654 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:24:04,655 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 18:24:04,656 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 18:24:04,687 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 18:24:04,687 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 18:24:04,687 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 18:24:04,688 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 18:24:04,689 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:24:04,689 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 18:24:04,689 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 18:24:04,693 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 18:24:04,745 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 18:24:04,746 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 18:24:04,746 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 18:24:04,747 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:24:04,751 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:09,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:11,089 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 18:24:11,089 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 18:24:11,089 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:24:11,089 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:24:11,091 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:24:11,094 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:24:11,095 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:24:11,097 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 18:24:11,099 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 18:24:11,159 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 18:24:11,159 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 18:24:11,159 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 18:24:11,160 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:24:11,160 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 18:24:11,160 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 18:24:11,162 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 18:24:11,163 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 18:24:11,215 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 18:24:11,215 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 18:24:11,216 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 18:24:11,216 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:24:12,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:24:12,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:24:12,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:14,909 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 18:24:14,909 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 18:24:14,910 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:24:14,910 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:24:14,911 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:24:14,912 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:24:14,914 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:24:14,918 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 18:24:14,919 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 18:24:14,950 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 18:24:14,950 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 18:24:14,950 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 18:24:14,951 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 18:24:14,952 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 18:24:14,953 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:24:14,953 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 18:24:14,953 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 18:24:15,008 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 18:24:15,009 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 18:24:15,009 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 18:24:15,010 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:24:15,014 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:20,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:22,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:26,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:27,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:24:27,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:24:32,138 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:32,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:37,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:42,872 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:24:42,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:24:42,913 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:43,090 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:48,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:52,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:24:53,880 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:24:57,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:24:57,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:24:58,996 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:02,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:04,880 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:10,880 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:12,873 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:25:12,873 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:25:12,913 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:16,092 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:16,270 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 18:25:16,270 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 18:25:16,271 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 18:25:16,271 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 18:25:16,272 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 18:25:16,273 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 18:25:16,277 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 18:25:16,281 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 18:25:16,283 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 18:25:17,169 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:25:19,171 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:25:21,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:22,880 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:23,060 DEBUG SenderThread:1120898 [sender.py:send():391] send: config +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:25:23,062 WARNING SenderThread:1120898 [sender.py:send_metric():1417] Seen metric with glob (shouldn't happen) +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 18:25:23,062 DEBUG SenderThread:1120898 [sender.py:send():391] send: config +2024-09-16 18:25:25,178 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:25:27,059 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:27,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:25:27,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:25:32,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:33,059 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:39,068 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:40,193 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/config.yaml +2024-09-16 18:25:42,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:25:42,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:25:42,915 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:44,995 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:50,060 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:52,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:25:55,060 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:25:57,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:25:57,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:26:00,987 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:02,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:06,060 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:11,061 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:12,874 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:26:12,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:26:12,915 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:16,283 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 18:26:16,286 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:26:16,290 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:22,061 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:22,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:27,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:26:27,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:26:29,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:32,882 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:35,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:41,061 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:42,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:26:42,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:26:42,915 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:46,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:46,288 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:26:51,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:52,882 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:26:56,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:26:57,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:26:57,875 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:27:01,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:02,882 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:06,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:11,062 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:12,875 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:27:12,876 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:27:12,916 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:16,291 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:27:16,295 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:22,063 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:22,882 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:27,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:27:27,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:27:28,043 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:32,882 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:33,063 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:39,063 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:42,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:27:42,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:27:42,917 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:44,095 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:46,293 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:27:49,295 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:52,884 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:27:55,064 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:27:57,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:27:57,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:28:00,887 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:02,885 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:06,065 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:11,065 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:12,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:28:12,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:28:12,917 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:16,065 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:16,295 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:28:21,065 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:22,885 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:26,066 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:27,876 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:28:27,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:28:31,066 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:32,885 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:37,066 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:42,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:28:42,877 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:28:42,917 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:43,042 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:46,297 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:28:48,299 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:52,888 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:28:54,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:28:57,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:28:57,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:29:00,613 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:02,888 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:06,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:11,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:12,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:29:12,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:29:12,918 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:16,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:16,300 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:29:21,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:22,889 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:26,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:27,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:29:27,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:29:31,067 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:32,889 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:36,068 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:41,068 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:42,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:29:42,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:29:42,918 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:46,071 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:46,305 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:29:51,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:52,889 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:29:56,071 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:29:57,877 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:29:57,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:30:01,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:02,889 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:06,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:11,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:12,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:30:12,878 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:30:12,918 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:16,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:16,304 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:30:22,072 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:22,889 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:27,073 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:27,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:30:27,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:30:32,073 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:32,890 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:37,073 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:42,074 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:42,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:30:42,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:30:42,919 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:46,309 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:30:47,311 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:52,891 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:30:53,074 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:30:57,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:30:57,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:30:59,015 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:02,892 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:04,074 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:09,074 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:12,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:31:12,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:31:12,919 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:15,029 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:16,309 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:31:20,311 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:22,892 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:26,075 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:27,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:31:27,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:31:32,075 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:32,892 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:37,075 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:42,075 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:42,878 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:31:42,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:31:42,919 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:46,311 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:31:47,313 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:52,892 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:31:53,076 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:31:57,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:31:57,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:31:58,098 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:02,407 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:32:02,411 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:32:02,411 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:32:02,412 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:32:02,412 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:32:02,412 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:32:02,412 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:32:02,416 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:32:02,563 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_created():271] file/dir created: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:32:02,892 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:03,417 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:09,076 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:12,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:32:12,879 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:32:12,920 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:14,079 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:16,313 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:32:19,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:22,893 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:25,077 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:27,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:32:27,880 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:32:30,106 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:30,587 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/config.yaml +2024-09-16 18:32:32,893 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:36,077 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:41,077 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:42,879 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:32:42,880 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:32:42,920 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:46,077 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:46,315 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:32:52,077 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:32:52,893 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:32:57,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:32:57,882 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:33:00,059 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:02,893 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:05,078 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:11,078 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:12,881 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:33:12,882 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:33:12,922 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:16,106 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:16,318 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:33:17,509 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:33:17,514 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,514 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,514 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: metric +2024-09-16 18:33:17,515 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:33:17,516 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:33:17,519 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:33:17,628 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:33:19,730 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:33:21,520 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:26,109 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:27,114 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:28,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:33:28,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:33:32,336 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:36,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:37,430 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:38,093 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/config.yaml +2024-09-16 18:33:43,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:33:43,120 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:33:43,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:46,320 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:33:46,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:48,417 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:54,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:33:56,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:33:58,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:33:58,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:33:59,616 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:05,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:06,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:10,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:13,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:34:13,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:34:16,298 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:16,322 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:34:16,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:21,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:26,415 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:27,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:28,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:34:28,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:34:32,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:36,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:37,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:42,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:43,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:34:43,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:34:46,325 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:34:46,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:48,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:53,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:34:56,418 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:34:58,120 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:34:58,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:35:00,667 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:06,418 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:06,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:11,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:13,121 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:35:13,121 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:35:16,327 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:35:16,418 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:16,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:21,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:26,418 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:26,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:28,121 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:35:28,122 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:35:31,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:36,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:36,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:41,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:43,121 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:35:43,122 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:35:46,329 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:35:46,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:47,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:52,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:35:56,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:35:58,121 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:35:58,122 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:35:59,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:04,425 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:06,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:10,425 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:13,122 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:36:13,123 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:36:16,320 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:16,331 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:36:16,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:21,425 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:26,419 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:26,426 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:28,122 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:36:28,123 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:36:32,426 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:36,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:38,426 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:43,122 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:36:43,123 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:36:44,272 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:46,337 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:36:46,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:49,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:54,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:36:56,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:36:58,123 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:36:58,123 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:37:00,068 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:05,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:06,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:11,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:13,123 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:37:13,124 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:37:16,336 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:37:16,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:16,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:21,516 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:26,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:26,516 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:28,123 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:37:28,124 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:37:31,516 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:36,420 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:36,516 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:41,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:43,123 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:37:43,124 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:37:46,340 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:37:46,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:46,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:51,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:56,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:37:57,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:37:58,123 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:37:58,124 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:38:02,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:06,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:07,523 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:12,528 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:13,124 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:38:13,124 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:38:16,341 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:38:16,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:18,526 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:24,526 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:26,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:28,124 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:38:28,125 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:38:30,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:35,527 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:36,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:41,526 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:43,130 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:38:43,131 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:38:46,343 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:38:46,421 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:46,527 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:51,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:56,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:38:57,527 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:38:58,126 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:38:58,127 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:39:02,527 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:06,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:08,527 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:13,126 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:39:13,127 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:39:14,306 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:16,345 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:39:16,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:19,528 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:25,528 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:26,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:28,126 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:39:28,127 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:39:31,342 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:36,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:36,528 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:42,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:43,132 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:39:43,132 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:39:46,347 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:39:46,422 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:48,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:53,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:39:56,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:39:58,128 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:39:58,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:39:59,747 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:05,530 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:06,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:11,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:13,128 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:40:13,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:40:15,308 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:40:15,310 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:40:15,311 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:40:15,315 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:40:15,485 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:40:16,350 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:40:16,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:17,529 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:22,530 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:26,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:28,128 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:40:28,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:40:30,275 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:35,530 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:36,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:40,532 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:43,128 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:40:43,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:40:46,304 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:46,352 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:40:46,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:51,531 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:56,423 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:40:56,532 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:40:58,128 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:40:58,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:41:01,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:06,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:41:07,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:12,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:13,129 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:41:13,129 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:41:16,354 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:41:16,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:41:17,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:22,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:26,475 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:41:27,545 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:28,149 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:41:28,150 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:41:30,374 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:41:30,376 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:41:30,377 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:41:30,379 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:41:30,551 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:41:32,656 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:41:33,381 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:38,382 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:39,144 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:41:44,145 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:46,356 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:41:49,358 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:49,372 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:41:49,373 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:41:49,413 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:41:54,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:41:59,361 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:00,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:04,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:42:04,367 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:42:05,517 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:09,361 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:11,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:16,359 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:42:17,360 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:19,362 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:19,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:42:19,367 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:42:22,542 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:28,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:29,362 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:34,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:42:34,367 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:42:34,581 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:39,362 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:40,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:45,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:46,361 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:42:49,363 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:42:49,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:42:49,367 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:42:50,593 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:56,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:42:59,363 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:02,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:04,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:43:04,367 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:43:07,549 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:09,363 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:13,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:16,363 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:43:19,363 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:19,365 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:19,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:43:19,370 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:43:24,379 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:29,363 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:29,379 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:34,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:43:34,368 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:43:34,533 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:39,364 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:40,379 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:46,367 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:43:46,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:49,364 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:43:49,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:43:49,368 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:43:51,535 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:57,380 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:43:59,364 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:03,380 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:04,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:44:04,368 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:44:08,585 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:09,365 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:14,380 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:16,368 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:44:19,365 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:19,368 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:44:19,368 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:44:19,506 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:25,387 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:29,365 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:31,387 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:34,368 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:44:34,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:44:36,553 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:39,365 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:42,388 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:46,370 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:44:48,372 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:49,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:49,368 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:44:49,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:44:53,601 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:44:59,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:44:59,392 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:04,368 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:45:04,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:45:04,575 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:09,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:45:10,392 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:15,393 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:16,372 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:45:19,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:45:19,368 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:45:19,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:45:20,540 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:26,393 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:29,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:45:32,393 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:34,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:45:34,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:45:37,608 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:39,366 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:45:43,400 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:46,375 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:45:49,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:45:49,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:45:49,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:45:49,596 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:55,398 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:45:59,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:01,399 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:04,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:46:04,369 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:46:06,530 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:09,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:12,399 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:16,377 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:46:18,379 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:19,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:19,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:46:19,370 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:46:23,549 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:29,367 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:29,399 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:34,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:46:34,370 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:46:34,591 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:39,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:40,400 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:45,400 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:46,379 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:46:49,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:46:49,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:46:49,371 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:46:50,573 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:56,401 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:46:59,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:02,401 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:04,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:47:04,371 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:47:07,507 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:09,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:13,401 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:16,381 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:47:19,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:19,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:47:19,371 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:47:19,580 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:25,401 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:29,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:31,401 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:34,371 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:47:34,371 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:47:36,515 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:39,369 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:42,402 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:46,384 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:47:48,385 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:49,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:49,371 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:47:49,372 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:47:53,551 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:47:59,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:47:59,402 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:04,371 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:48:04,371 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:48:04,577 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:09,370 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:10,402 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:15,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:16,386 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:48:19,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:48:19,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:48:19,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:20,570 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:26,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:28,503 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:48:28,505 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:48:28,505 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:48:28,508 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:48:29,352 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:48:29,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:31,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:34,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:48:34,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:48:36,555 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:39,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:42,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:46,388 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:48:48,390 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:49,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:48:49,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:48:49,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:53,520 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:48:59,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:48:59,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:04,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:49:04,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:49:04,525 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:09,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:49:10,404 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:15,404 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:16,390 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:49:19,375 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:49:19,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:49:19,416 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:49:20,600 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:26,404 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:29,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:49:31,405 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:34,376 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:49:34,376 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:49:36,545 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:39,377 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:49:42,405 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:43,681 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:49:43,684 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:49:43,684 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:49:43,687 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:49:44,419 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:49:46,393 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:49:46,490 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:49:48,394 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:53,313 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:49:53,314 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:49:53,315 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:49:53,482 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:49:58,483 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:03,484 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:03,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:08,314 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:50:08,315 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:50:08,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:13,768 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:13,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:16,395 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:50:19,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:23,314 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:50:23,315 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:50:23,768 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:24,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:29,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:33,768 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:34,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:38,314 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:50:38,315 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:50:40,777 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:43,769 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:45,778 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:46,397 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:50:51,778 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:50:53,314 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:50:53,315 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:50:53,769 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:50:57,778 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:02,778 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:03,769 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:07,778 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:08,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:51:08,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:51:12,779 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:13,769 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:16,401 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:51:18,403 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:23,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:51:23,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:51:23,530 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:23,769 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:28,779 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:33,770 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:33,779 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:38,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:51:38,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:51:39,779 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:43,770 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:44,780 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:46,402 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:51:50,780 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:51:53,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:51:53,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:51:53,770 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:51:55,780 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:01,780 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:03,770 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:07,780 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:08,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:52:08,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:52:12,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:13,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:16,404 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:52:18,406 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:23,315 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:52:23,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:52:23,476 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:23,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:28,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:33,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:33,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:38,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:52:38,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:52:38,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:43,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:43,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:46,406 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:52:48,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:53,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:52:53,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:52:53,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:52:54,781 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:52:59,782 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:03,771 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:04,785 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:08,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:53:08,317 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:53:10,783 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:13,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:15,783 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:16,408 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:53:20,783 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:23,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:53:23,316 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:53:23,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:25,784 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:30,784 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:33,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:36,784 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:38,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:53:38,317 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:53:41,784 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:43,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:46,410 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:53:47,412 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:52,785 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:53:53,316 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:53:53,317 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:53:53,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:53:58,785 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:03,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:03,785 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:08,317 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:54:08,317 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:54:08,787 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:13,772 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:13,787 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:16,413 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:54:18,788 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:23,317 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:54:23,318 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:54:23,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:24,788 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:29,788 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:33,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:34,788 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:38,320 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:54:38,321 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:54:40,788 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:43,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:45,789 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:46,415 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:54:51,789 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:54:53,320 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:54:53,321 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:54:53,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:54:57,789 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:02,789 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:03,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:07,793 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:08,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:55:08,321 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:55:13,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:13,792 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:16,417 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:55:18,792 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:23,320 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:55:23,321 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:55:23,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:24,792 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:30,792 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:33,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:35,793 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:38,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:55:38,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:55:40,793 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:43,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:46,419 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:55:46,424 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:51,793 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:55:53,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:55:53,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:55:53,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:55:57,793 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:02,794 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:03,774 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:07,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:08,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:56:08,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:56:13,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:13,799 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:16,422 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:56:18,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:23,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:56:23,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:56:23,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:23,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:29,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:33,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:34,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:38,321 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:56:38,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:56:39,800 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:43,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:44,155 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:56:44,157 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:56:44,157 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:56:44,160 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:56:44,352 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:56:45,161 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:46,424 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:56:50,801 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:56:53,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:56:53,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:56:53,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:56:56,801 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:01,801 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:03,775 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:06,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:08,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:57:08,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:57:11,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:13,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:16,426 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:57:17,428 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:22,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:23,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:57:23,322 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:57:23,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:28,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:33,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:33,803 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:38,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:57:38,323 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:57:39,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:43,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:44,803 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:46,428 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:57:49,803 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:53,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:57:53,323 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:57:53,776 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:57:55,803 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:57:58,996 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 18:57:58,999 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 18:57:58,999 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 18:57:59,001 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 18:57:59,417 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 18:58:00,419 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 18:58:01,003 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:06,003 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:07,284 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:58:08,322 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:58:08,323 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:58:11,543 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:16,442 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:58:17,444 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:18,187 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:58:23,196 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:23,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:58:23,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:58:28,180 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:58:29,196 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:34,196 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:38,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:58:38,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:58:38,323 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:58:39,514 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:45,197 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:46,433 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:58:48,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:58:51,197 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:53,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:58:53,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:58:56,547 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:58:58,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:02,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:08,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:08,198 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:08,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:59:08,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:59:13,198 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:16,438 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:59:18,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:19,198 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:23,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:59:23,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:59:24,458 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:28,181 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:30,199 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:35,199 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:38,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:38,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:59:38,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:59:40,498 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:46,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:46,437 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 18:59:48,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 18:59:51,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:53,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 18:59:53,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 18:59:56,520 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 18:59:58,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:02,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:07,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:08,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:08,323 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:00:08,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:00:12,523 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:16,440 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:00:18,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:18,200 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:23,201 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:23,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:00:23,324 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:00:28,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:28,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:34,204 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:38,182 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:38,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:00:38,325 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:00:39,460 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:45,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:46,442 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:00:48,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:00:51,204 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:53,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:00:53,325 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:00:56,497 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:00:58,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:02,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:07,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:08,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:08,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:01:08,325 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:01:12,534 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:16,444 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:01:18,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:18,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:23,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:23,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:01:23,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:01:28,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:29,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:34,205 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:38,183 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:38,324 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:01:38,325 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:01:39,491 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:45,206 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:46,446 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:01:48,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:01:51,206 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:53,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:01:53,325 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:01:56,477 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:01:58,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:02,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:07,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:08,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:08,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:02:08,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:02:12,503 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:16,449 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:02:18,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:18,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:23,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:23,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:02:23,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:02:28,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:29,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:34,207 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:38,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:38,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:02:38,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:02:39,492 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:45,208 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:46,451 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:02:48,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:02:50,208 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:53,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:02:53,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:02:55,538 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:02:58,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:01,208 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:06,209 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:08,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:08,325 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:03:08,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:03:11,543 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:16,453 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:03:17,455 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:18,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:23,215 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:23,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:03:23,326 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:03:28,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:28,215 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:33,216 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:38,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:38,216 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:38,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:03:38,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:03:43,217 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:46,455 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:03:48,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:03:49,217 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:53,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:03:53,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:03:54,467 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:03:58,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:00,222 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:06,221 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:08,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:08,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:04:08,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:04:11,552 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:16,458 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:04:17,459 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:18,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:23,221 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:23,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:04:23,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:04:28,188 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:28,222 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:34,222 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:38,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:38,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:04:38,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:04:39,492 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:45,224 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:46,460 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:04:48,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:04:50,224 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:53,326 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:04:53,327 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:04:55,531 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:04:58,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:01,225 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:06,915 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:08,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:08,327 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:05:08,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:05:12,510 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:16,462 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:05:16,485 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 19:05:16,487 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 19:05:16,488 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:05:16,491 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:05:16,877 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 19:05:18,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:18,492 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:23,327 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:05:23,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:05:24,481 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:28,189 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:30,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:35,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:38,191 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:38,327 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:05:38,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:05:40,969 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:46,150 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:46,466 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:05:48,190 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:05:52,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:53,327 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:05:53,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:05:57,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:05:58,190 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:02,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:08,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:08,190 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:08,327 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:06:08,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:06:13,151 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:16,467 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:06:18,190 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:18,468 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:23,328 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:06:23,328 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:06:23,479 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:28,216 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:29,152 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:31,789 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 19:06:31,792 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 19:06:31,792 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:31,795 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:31,944 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 19:06:34,802 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:35,066 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/output.log +2024-09-16 19:06:39,803 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:39,962 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:39,964 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:06:39,965 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:06:45,184 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:46,469 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:06:50,471 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:50,691 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:06:54,762 DEBUG SenderThread:1120898 [sender.py:send():391] send: telemetry +2024-09-16 19:06:54,804 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: partial_history +2024-09-16 19:06:54,805 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 19:06:54,805 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 19:06:54,805 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 19:06:54,807 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 19:06:54,808 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 19:06:54,808 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,811 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 19:06:54,811 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 19:06:54,814 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,814 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,818 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,818 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,822 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,822 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,826 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,826 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,830 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,830 DEBUG SenderThread:1120898 [sender.py:send():391] send: history +2024-09-16 19:06:54,830 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: summary_record +2024-09-16 19:06:54,833 INFO SenderThread:1120898 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end +2024-09-16 19:06:54,877 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 19:06:54,877 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 19:06:54,878 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 19:06:54,878 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:06:54,959 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:06:54,960 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:06:55,391 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/wandb-summary.json +2024-09-16 19:06:55,705 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:06:56,392 INFO Thread-12 :1120898 [dir_watcher.py:_on_file_modified():288] file/dir modified: /nfs/production/literature/amina-mardiyyah/NER/ewc_stabilised/wandb/run-20240916_182041-ye308qxr/files/config.yaml +2024-09-16 19:07:00,691 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:01,700 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:02,733 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 19:07:02,733 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 19:07:02,733 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 19:07:02,733 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 19:07:02,735 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 19:07:02,738 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 19:07:02,739 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 19:07:02,740 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 19:07:02,743 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 19:07:02,773 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 19:07:02,773 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 19:07:02,773 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 19:07:02,774 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 19:07:02,775 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 19:07:02,776 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 19:07:02,776 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 19:07:02,776 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 19:07:02,835 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 19:07:02,836 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 19:07:02,836 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 19:07:02,837 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:07:06,700 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:09,959 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:07:09,960 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:07:10,691 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:12,700 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:17,701 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:20,691 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:21,411 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 19:07:21,412 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 19:07:21,412 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 19:07:21,412 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 19:07:21,413 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 19:07:21,416 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 19:07:21,417 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 19:07:21,419 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 19:07:21,420 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 19:07:21,532 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: pause +2024-09-16 19:07:21,532 INFO HandlerThread:1120898 [handler.py:handle_request_pause():728] stopping system metrics thread +2024-09-16 19:07:21,532 INFO HandlerThread:1120898 [system_monitor.py:finish():203] Stopping system monitor +2024-09-16 19:07:21,533 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined cpu monitor +2024-09-16 19:07:21,533 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined disk monitor +2024-09-16 19:07:21,534 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 19:07:21,535 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-16 19:07:21,535 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-16 19:07:21,595 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined gpu monitor +2024-09-16 19:07:21,596 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined memory monitor +2024-09-16 19:07:21,596 INFO HandlerThread:1120898 [interfaces.py:finish():200] Joined network monitor +2024-09-16 19:07:21,597 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:07:23,599 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:24,959 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:07:24,960 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:07:28,701 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:30,692 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:33,701 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:38,701 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:39,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:07:39,960 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:07:40,692 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:44,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:49,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:50,692 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:07:54,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:07:54,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:07:54,960 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:08:00,692 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:00,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:05,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:09,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:08:09,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:08:10,693 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:10,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:15,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:20,693 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:20,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:24,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:08:24,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:08:25,702 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:30,693 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:30,703 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:35,703 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:39,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:08:39,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:08:40,693 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:40,703 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:43,071 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: resume +2024-09-16 19:08:43,071 INFO HandlerThread:1120898 [handler.py:handle_request_resume():719] starting system metrics thread +2024-09-16 19:08:43,071 INFO HandlerThread:1120898 [system_monitor.py:start():194] Starting system monitor +2024-09-16 19:08:43,072 INFO SystemMonitor:1120898 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-16 19:08:43,073 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started cpu monitoring +2024-09-16 19:08:43,075 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started disk monitoring +2024-09-16 19:08:43,077 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started gpu monitoring +2024-09-16 19:08:43,080 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started memory monitoring +2024-09-16 19:08:43,082 INFO SystemMonitor:1120898 [interfaces.py:start():188] Started network monitoring +2024-09-16 19:08:45,704 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:50,705 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:08:52,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:08:54,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:08:54,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:08:56,147 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:01,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:02,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:06,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:09,960 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:09:09,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:09:12,185 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:12,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:17,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:22,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:22,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:24,961 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:09:24,961 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:09:28,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:32,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:33,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:38,934 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:39,961 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:09:39,962 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:09:42,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:43,083 DEBUG SystemMonitor:1120898 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-16 19:09:43,086 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats +2024-09-16 19:09:44,088 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:49,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:09:52,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:09:54,961 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:09:54,962 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:09:55,149 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:10:00,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:10:02,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:10:05,935 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:10:09,961 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: stop_status +2024-09-16 19:10:09,962 DEBUG SenderThread:1120898 [sender.py:send_request():418] send_request: stop_status +2024-09-16 19:10:11,169 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: status_report +2024-09-16 19:10:12,936 DEBUG HandlerThread:1120898 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-16 19:10:13,088 DEBUG SenderThread:1120898 [sender.py:send():391] send: stats