diff --git "a/diffusion_infer.log" "b/diffusion_infer.log" new file mode 100644--- /dev/null +++ "b/diffusion_infer.log" @@ -0,0 +1,28951 @@ +2024-01-10 13:57:08 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 13:57:08 | INFO | stdout | +2024-01-10 13:57:08 | ERROR | stderr | /fs01/home/tianleli/ImagenHub/src/imagen_hub/pipelines/blended_diffusion/blended_diffusion_src/CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended +2024-01-10 13:57:08 | ERROR | stderr | warnings.warn("PyTorch version 1.7.1 or higher is recommended") +2024-01-10 13:57:15 | ERROR | stderr | usage: model_worker.py [-h] [--host HOST] [--port PORT] +2024-01-10 13:57:15 | ERROR | stderr | [--worker-address WORKER_ADDRESS] +2024-01-10 13:57:15 | ERROR | stderr | [--controller-address CONTROLLER_ADDRESS] +2024-01-10 13:57:15 | ERROR | stderr | [--model-path MODEL_PATH] [--revision REVISION] +2024-01-10 13:57:15 | ERROR | stderr | [--device {cpu,cuda,mps,xpu,npu}] [--gpus GPUS] +2024-01-10 13:57:15 | ERROR | stderr | [--num-gpus NUM_GPUS] [--max-gpu-memory MAX_GPU_MEMORY] +2024-01-10 13:57:15 | ERROR | stderr | [--dtype {float32,float16,bfloat16}] [--load-8bit] +2024-01-10 13:57:15 | ERROR | stderr | [--cpu-offloading] [--gptq-ckpt GPTQ_CKPT] +2024-01-10 13:57:15 | ERROR | stderr | [--gptq-wbits {2,3,4,8,16}] +2024-01-10 13:57:15 | ERROR | stderr | [--gptq-groupsize GPTQ_GROUPSIZE] [--gptq-act-order] +2024-01-10 13:57:15 | ERROR | stderr | [--awq-ckpt AWQ_CKPT] [--awq-wbits {4,16}] +2024-01-10 13:57:15 | ERROR | stderr | [--awq-groupsize AWQ_GROUPSIZE] [--enable-exllama] +2024-01-10 13:57:15 | ERROR | stderr | [--exllama-max-seq-len EXLLAMA_MAX_SEQ_LEN] +2024-01-10 13:57:15 | ERROR | stderr | [--exllama-gpu-split EXLLAMA_GPU_SPLIT] +2024-01-10 13:57:15 | ERROR | stderr | [--exllama-cache-8bit] [--enable-xft] +2024-01-10 13:57:15 | ERROR | stderr | [--xft-max-seq-len XFT_MAX_SEQ_LEN] +2024-01-10 13:57:15 | ERROR | stderr | [--xft-dtype {fp16,bf16,int8,bf16_fp16,bf16_int8}] +2024-01-10 13:57:15 | ERROR | stderr | [--model-names MODEL_NAMES] +2024-01-10 13:57:15 | ERROR | stderr | [--conv-template CONV_TEMPLATE] [--embed-in-truncate] +2024-01-10 13:57:15 | ERROR | stderr | [--limit-worker-concurrency LIMIT_WORKER_CONCURRENCY] +2024-01-10 13:57:15 | ERROR | stderr | [--stream-interval STREAM_INTERVAL] [--no-register] +2024-01-10 13:57:15 | ERROR | stderr | [--seed SEED] [--debug DEBUG] [--ssl] +2024-01-10 13:57:15 | ERROR | stderr | model_worker.py: error: argument --host: expected one argument +2024-01-10 13:59:04 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 13:59:04 | INFO | stdout | +2024-01-10 13:59:04 | ERROR | stderr | /fs01/home/tianleli/ImagenHub/src/imagen_hub/pipelines/blended_diffusion/blended_diffusion_src/CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended +2024-01-10 13:59:04 | ERROR | stderr | warnings.warn("PyTorch version 1.7.1 or higher is recommended") +2024-01-10 13:59:08 | ERROR | stderr | usage: model_worker.py [-h] [--host HOST] [--port PORT] +2024-01-10 13:59:08 | ERROR | stderr | [--worker-address WORKER_ADDRESS] +2024-01-10 13:59:08 | ERROR | stderr | [--controller-address CONTROLLER_ADDRESS] +2024-01-10 13:59:08 | ERROR | stderr | [--model-path MODEL_PATH] [--revision REVISION] +2024-01-10 13:59:08 | ERROR | stderr | [--device {cpu,cuda,mps,xpu,npu}] [--gpus GPUS] +2024-01-10 13:59:08 | ERROR | stderr | [--num-gpus NUM_GPUS] [--max-gpu-memory MAX_GPU_MEMORY] +2024-01-10 13:59:08 | ERROR | stderr | [--dtype {float32,float16,bfloat16}] [--load-8bit] +2024-01-10 13:59:08 | ERROR | stderr | [--cpu-offloading] [--gptq-ckpt GPTQ_CKPT] +2024-01-10 13:59:08 | ERROR | stderr | [--gptq-wbits {2,3,4,8,16}] +2024-01-10 13:59:08 | ERROR | stderr | [--gptq-groupsize GPTQ_GROUPSIZE] [--gptq-act-order] +2024-01-10 13:59:08 | ERROR | stderr | [--awq-ckpt AWQ_CKPT] [--awq-wbits {4,16}] +2024-01-10 13:59:08 | ERROR | stderr | [--awq-groupsize AWQ_GROUPSIZE] [--enable-exllama] +2024-01-10 13:59:08 | ERROR | stderr | [--exllama-max-seq-len EXLLAMA_MAX_SEQ_LEN] +2024-01-10 13:59:08 | ERROR | stderr | [--exllama-gpu-split EXLLAMA_GPU_SPLIT] +2024-01-10 13:59:08 | ERROR | stderr | [--exllama-cache-8bit] [--enable-xft] +2024-01-10 13:59:08 | ERROR | stderr | [--xft-max-seq-len XFT_MAX_SEQ_LEN] +2024-01-10 13:59:08 | ERROR | stderr | [--xft-dtype {fp16,bf16,int8,bf16_fp16,bf16_int8}] +2024-01-10 13:59:08 | ERROR | stderr | [--model-names MODEL_NAMES] +2024-01-10 13:59:08 | ERROR | stderr | [--conv-template CONV_TEMPLATE] [--embed-in-truncate] +2024-01-10 13:59:08 | ERROR | stderr | [--limit-worker-concurrency LIMIT_WORKER_CONCURRENCY] +2024-01-10 13:59:08 | ERROR | stderr | [--stream-interval STREAM_INTERVAL] [--no-register] +2024-01-10 13:59:08 | ERROR | stderr | [--seed SEED] [--debug DEBUG] [--ssl] +2024-01-10 13:59:08 | ERROR | stderr | model_worker.py: error: argument --host: expected one argument +2024-01-10 14:01:21 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 14:01:21 | INFO | stdout | +2024-01-10 14:01:21 | ERROR | stderr | /fs01/home/tianleli/ImagenHub/src/imagen_hub/pipelines/blended_diffusion/blended_diffusion_src/CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended +2024-01-10 14:01:21 | ERROR | stderr | warnings.warn("PyTorch version 1.7.1 or higher is recommended") +2024-01-10 14:01:26 | ERROR | stderr | Loading pipeline components...: 0%| | 0/6 [00:00 +2024-01-10 14:05:43 | ERROR | stderr | args, worker = create_model_worker() +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 362, in create_model_worker +2024-01-10 14:05:43 | ERROR | stderr | worker = ModelWorker( +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 86, in __init__ +2024-01-10 14:05:43 | ERROR | stderr | self.model, self.tokenizer = load_model( +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 332, in load_model +2024-01-10 14:05:43 | ERROR | stderr | model, tokenizer = adapter.load_model(model_path, kwargs) +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 2043, in load_model +2024-01-10 14:05:43 | ERROR | stderr | model = imagen_hub.load(model_name) +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/__init__.py", line 86, in get_model +2024-01-10 14:05:43 | ERROR | stderr | model_instance = model_class() +2024-01-10 14:05:43 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/pixart_alpha.py", line 19, in __init__ +2024-01-10 14:05:43 | ERROR | stderr | self.pipe = PixArtAlphaPipeline.from_pretrained( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1090, in from_pretrained +2024-01-10 14:05:43 | ERROR | stderr | cached_folder = cls.download( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1856, in download +2024-01-10 14:05:43 | ERROR | stderr | cached_folder = snapshot_download( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:05:43 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 308, in snapshot_download +2024-01-10 14:05:43 | ERROR | stderr | thread_map( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map +2024-01-10 14:05:43 | ERROR | stderr | return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map +2024-01-10 14:05:43 | ERROR | stderr | return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs)) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/std.py", line 1182, in __iter__ +2024-01-10 14:05:43 | ERROR | stderr | for obj in iterable: +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 609, in result_iterator +2024-01-10 14:05:43 | ERROR | stderr | yield fs.pop().result() +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 446, in result +2024-01-10 14:05:43 | ERROR | stderr | return self.__get_result() +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result +2024-01-10 14:05:43 | ERROR | stderr | raise self._exception +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/thread.py", line 58, in run +2024-01-10 14:05:43 | ERROR | stderr | result = self.fn(*self.args, **self.kwargs) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 283, in _inner_hf_hub_download +2024-01-10 14:05:43 | ERROR | stderr | return hf_hub_download( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:05:43 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1457, in hf_hub_download +2024-01-10 14:05:43 | ERROR | stderr | http_get( +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 527, in http_get +2024-01-10 14:05:43 | ERROR | stderr | temp_file.write(chunk) +2024-01-10 14:05:43 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/tempfile.py", line 478, in func_wrapper +2024-01-10 14:05:43 | ERROR | stderr | return func(*args, **kwargs) +2024-01-10 14:05:43 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:10:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:10:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:10:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:10:14 | ERROR | stderr | response.begin() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:10:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:10:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:10:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:10:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:10:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:10:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:10:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:10:14 | ERROR | stderr | raise value +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:10:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:10:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:10:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:10:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:10:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:10:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:10:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:10:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:10:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:10:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:10:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:14 | ERROR | stderr | +2024-01-10 14:10:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:10:14 | ERROR | stderr | self.flush() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:10:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:10:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:14 | ERROR | stderr | Call stack: +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:10:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:10:14 | ERROR | stderr | self.run() +2024-01-10 14:10:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:10:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:10:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:10:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:10:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:10:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:10:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:10:14 | ERROR | stderr | Arguments: () +2024-01-10 14:10:24 | ERROR | stderr | --- Logging error --- +2024-01-10 14:10:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:10:24 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:10:24 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:10:24 | ERROR | stderr | response.begin() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:10:24 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:10:24 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:10:24 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:10:24 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:10:24 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:10:24 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:10:24 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:10:24 | ERROR | stderr | raise value +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:10:24 | ERROR | stderr | response = self._make_request( +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:10:24 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:10:24 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:10:24 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:10:24 | ERROR | stderr | ret = requests.post( +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:10:24 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:10:24 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:10:24 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:10:24 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:10:24 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:10:24 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:24 | ERROR | stderr | +2024-01-10 14:10:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:10:24 | ERROR | stderr | self.flush() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:10:24 | ERROR | stderr | self.stream.flush() +2024-01-10 14:10:24 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:24 | ERROR | stderr | Call stack: +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:10:24 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:10:24 | ERROR | stderr | self.run() +2024-01-10 14:10:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:10:24 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:10:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:10:24 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:10:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:10:24 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:10:24 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:10:24 | ERROR | stderr | Arguments: () +2024-01-10 14:10:34 | ERROR | stderr | --- Logging error --- +2024-01-10 14:10:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:10:34 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:10:34 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:10:34 | ERROR | stderr | response.begin() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:10:34 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:10:34 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:10:34 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:10:34 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:10:34 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:10:34 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:10:34 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:10:34 | ERROR | stderr | raise value +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:10:34 | ERROR | stderr | response = self._make_request( +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:10:34 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:10:34 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:10:34 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:10:34 | ERROR | stderr | ret = requests.post( +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:10:34 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:10:34 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:10:34 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:10:34 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:10:34 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:10:34 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:34 | ERROR | stderr | +2024-01-10 14:10:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:10:34 | ERROR | stderr | self.flush() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:10:34 | ERROR | stderr | self.stream.flush() +2024-01-10 14:10:34 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:34 | ERROR | stderr | Call stack: +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:10:34 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:10:34 | ERROR | stderr | self.run() +2024-01-10 14:10:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:10:34 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:10:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:10:34 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:10:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:10:34 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:10:34 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:10:34 | ERROR | stderr | Arguments: () +2024-01-10 14:10:44 | ERROR | stderr | --- Logging error --- +2024-01-10 14:10:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:10:44 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:10:44 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:10:44 | ERROR | stderr | response.begin() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:10:44 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:10:44 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:10:44 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:10:44 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:10:44 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:10:44 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:10:44 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:10:44 | ERROR | stderr | raise value +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:10:44 | ERROR | stderr | response = self._make_request( +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:10:44 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:10:44 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:10:44 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:10:44 | ERROR | stderr | ret = requests.post( +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:10:44 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:10:44 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:10:44 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:10:44 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:10:44 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:10:44 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:44 | ERROR | stderr | +2024-01-10 14:10:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:10:44 | ERROR | stderr | self.flush() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:10:44 | ERROR | stderr | self.stream.flush() +2024-01-10 14:10:44 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:44 | ERROR | stderr | Call stack: +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:10:44 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:10:44 | ERROR | stderr | self.run() +2024-01-10 14:10:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:10:44 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:10:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:10:44 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:10:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:10:44 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:10:44 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:10:44 | ERROR | stderr | Arguments: () +2024-01-10 14:10:54 | ERROR | stderr | --- Logging error --- +2024-01-10 14:10:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:10:54 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:10:54 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:10:54 | ERROR | stderr | response.begin() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:10:54 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:10:54 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:10:54 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:10:54 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:10:54 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:10:54 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:10:54 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:10:54 | ERROR | stderr | raise value +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:10:54 | ERROR | stderr | response = self._make_request( +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:10:54 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:10:54 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:10:54 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:10:54 | ERROR | stderr | ret = requests.post( +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:10:54 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:10:54 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:10:54 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:10:54 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:10:54 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:10:54 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:10:54 | ERROR | stderr | +2024-01-10 14:10:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:10:54 | ERROR | stderr | self.flush() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:10:54 | ERROR | stderr | self.stream.flush() +2024-01-10 14:10:54 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:10:54 | ERROR | stderr | Call stack: +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:10:54 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:10:54 | ERROR | stderr | self.run() +2024-01-10 14:10:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:10:54 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:10:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:10:54 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:10:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:10:54 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:10:54 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:10:54 | ERROR | stderr | Arguments: () +2024-01-10 14:11:04 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:04 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:04 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:04 | ERROR | stderr | response.begin() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:04 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:04 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:04 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:04 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:04 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:04 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:04 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:04 | ERROR | stderr | raise value +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:04 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:04 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:04 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:04 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:04 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:04 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:04 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:04 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:04 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:04 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:04 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:04 | ERROR | stderr | +2024-01-10 14:11:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:04 | ERROR | stderr | self.flush() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:04 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:04 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:04 | ERROR | stderr | Call stack: +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:04 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:04 | ERROR | stderr | self.run() +2024-01-10 14:11:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:04 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:04 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:04 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:04 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:04 | ERROR | stderr | Arguments: () +2024-01-10 14:11:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:14 | ERROR | stderr | response.begin() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:14 | ERROR | stderr | raise value +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:14 | ERROR | stderr | +2024-01-10 14:11:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:14 | ERROR | stderr | self.flush() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:14 | ERROR | stderr | Call stack: +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:14 | ERROR | stderr | self.run() +2024-01-10 14:11:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:14 | ERROR | stderr | Arguments: () +2024-01-10 14:11:20 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:20 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:20 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:20 | ERROR | stderr | response.begin() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:20 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:20 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:20 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:20 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:20 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:20 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:20 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:20 | ERROR | stderr | raise value +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:20 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:20 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:20 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:20 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:20 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:20 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:20 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:20 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:20 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:20 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:20 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:20 | ERROR | stderr | +2024-01-10 14:11:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:20 | ERROR | stderr | self.flush() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:20 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:20 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:20 | ERROR | stderr | Call stack: +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:20 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:20 | ERROR | stderr | self.run() +2024-01-10 14:11:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:20 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:20 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:20 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:20 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:20 | ERROR | stderr | Arguments: () +2024-01-10 14:11:24 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:24 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:24 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:24 | ERROR | stderr | response.begin() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:24 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:24 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:24 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:24 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:24 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:24 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:24 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:24 | ERROR | stderr | raise value +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:24 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:24 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:24 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:24 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:24 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:24 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:24 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:24 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:24 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:24 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:24 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:24 | ERROR | stderr | +2024-01-10 14:11:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:24 | ERROR | stderr | self.flush() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:24 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:24 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:24 | ERROR | stderr | Call stack: +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:24 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:24 | ERROR | stderr | self.run() +2024-01-10 14:11:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:24 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:24 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:24 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:24 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:24 | ERROR | stderr | Arguments: () +2024-01-10 14:11:30 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:30 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:30 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:30 | ERROR | stderr | response.begin() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:30 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:30 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:30 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:30 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:30 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:30 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:30 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:30 | ERROR | stderr | raise value +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:30 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:30 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:30 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:30 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:30 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:30 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:30 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:30 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:30 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:30 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:30 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:30 | ERROR | stderr | +2024-01-10 14:11:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:30 | ERROR | stderr | self.flush() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:30 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:30 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:30 | ERROR | stderr | Call stack: +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:30 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:30 | ERROR | stderr | self.run() +2024-01-10 14:11:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:30 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:30 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:30 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:30 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:30 | ERROR | stderr | Arguments: () +2024-01-10 14:11:34 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:34 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:34 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:34 | ERROR | stderr | response.begin() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:34 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:34 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:34 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:34 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:34 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:34 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:34 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:34 | ERROR | stderr | raise value +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:34 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:34 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:34 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:34 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:34 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:34 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:34 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:34 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:34 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:34 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:34 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:34 | ERROR | stderr | +2024-01-10 14:11:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:34 | ERROR | stderr | self.flush() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:34 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:34 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:34 | ERROR | stderr | Call stack: +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:34 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:34 | ERROR | stderr | self.run() +2024-01-10 14:11:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:34 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:34 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:34 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:34 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:34 | ERROR | stderr | Arguments: () +2024-01-10 14:11:40 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:40 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:40 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:40 | ERROR | stderr | response.begin() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:40 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:40 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:40 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:40 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:40 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:40 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:40 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:40 | ERROR | stderr | raise value +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:40 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:40 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:40 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:40 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:40 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:40 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:40 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:40 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:40 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:40 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:40 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:40 | ERROR | stderr | +2024-01-10 14:11:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:40 | ERROR | stderr | self.flush() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:40 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:40 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:40 | ERROR | stderr | Call stack: +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:40 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:40 | ERROR | stderr | self.run() +2024-01-10 14:11:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:40 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:40 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:40 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:40 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:40 | ERROR | stderr | Arguments: () +2024-01-10 14:11:44 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:44 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:44 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:44 | ERROR | stderr | response.begin() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:44 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:44 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:44 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:44 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:44 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:44 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:44 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:44 | ERROR | stderr | raise value +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:44 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:44 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:44 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:44 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:44 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:44 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:44 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:44 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:44 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:44 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:44 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:44 | ERROR | stderr | +2024-01-10 14:11:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:44 | ERROR | stderr | self.flush() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:44 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:44 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:44 | ERROR | stderr | Call stack: +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:44 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:44 | ERROR | stderr | self.run() +2024-01-10 14:11:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:44 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:44 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:44 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:44 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:44 | ERROR | stderr | Arguments: () +2024-01-10 14:11:50 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:50 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:50 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:50 | ERROR | stderr | response.begin() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:50 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:50 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:50 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:50 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:50 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:50 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:50 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:50 | ERROR | stderr | raise value +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:50 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:50 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:50 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:50 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:50 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:50 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:50 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:50 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:50 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:50 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:50 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:50 | ERROR | stderr | +2024-01-10 14:11:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:50 | ERROR | stderr | self.flush() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:50 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:50 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:50 | ERROR | stderr | Call stack: +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:50 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:50 | ERROR | stderr | self.run() +2024-01-10 14:11:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:50 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:50 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:50 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:50 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:50 | ERROR | stderr | Arguments: () +2024-01-10 14:11:54 | ERROR | stderr | --- Logging error --- +2024-01-10 14:11:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:11:54 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:11:54 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:11:54 | ERROR | stderr | response.begin() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:11:54 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:11:54 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:11:54 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:11:54 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:11:54 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:11:54 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:11:54 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:11:54 | ERROR | stderr | raise value +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:11:54 | ERROR | stderr | response = self._make_request( +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:11:54 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:11:54 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:11:54 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:11:54 | ERROR | stderr | ret = requests.post( +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:11:54 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:11:54 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:11:54 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:11:54 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:11:54 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:11:54 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:11:54 | ERROR | stderr | +2024-01-10 14:11:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:11:54 | ERROR | stderr | self.flush() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:11:54 | ERROR | stderr | self.stream.flush() +2024-01-10 14:11:54 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:11:54 | ERROR | stderr | Call stack: +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:11:54 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:11:54 | ERROR | stderr | self.run() +2024-01-10 14:11:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:11:54 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:11:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:11:54 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:11:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:11:54 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:11:54 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:11:54 | ERROR | stderr | Arguments: () +2024-01-10 14:12:00 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:00 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:00 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:00 | ERROR | stderr | response.begin() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:00 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:00 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:00 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:00 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:00 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:00 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:00 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:00 | ERROR | stderr | raise value +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:00 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:00 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:00 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:00 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:00 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:00 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:00 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:00 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:00 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:00 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:00 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:00 | ERROR | stderr | +2024-01-10 14:12:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:00 | ERROR | stderr | self.flush() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:00 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:00 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:00 | ERROR | stderr | Call stack: +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:00 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:00 | ERROR | stderr | self.run() +2024-01-10 14:12:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:00 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:00 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:00 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:00 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:00 | ERROR | stderr | Arguments: () +2024-01-10 14:12:04 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:04 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:04 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:04 | ERROR | stderr | response.begin() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:04 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:04 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:04 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:04 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:04 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:04 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:04 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:04 | ERROR | stderr | raise value +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:04 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:04 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:04 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:04 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:04 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:04 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:04 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:04 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:04 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:04 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:04 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:04 | ERROR | stderr | +2024-01-10 14:12:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:04 | ERROR | stderr | self.flush() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:04 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:04 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:04 | ERROR | stderr | Call stack: +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:04 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:04 | ERROR | stderr | self.run() +2024-01-10 14:12:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:04 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:04 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:04 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:04 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:04 | ERROR | stderr | Arguments: () +2024-01-10 14:12:10 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:10 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:10 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:10 | ERROR | stderr | response.begin() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:10 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:10 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:10 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:10 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:10 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:10 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:10 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:10 | ERROR | stderr | raise value +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:10 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:10 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:10 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:10 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:10 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:10 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:10 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:10 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:10 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:10 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:10 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:10 | ERROR | stderr | +2024-01-10 14:12:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:10 | ERROR | stderr | self.flush() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:10 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:10 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:10 | ERROR | stderr | Call stack: +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:10 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:10 | ERROR | stderr | self.run() +2024-01-10 14:12:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:10 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:10 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:10 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:10 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:10 | ERROR | stderr | Arguments: () +2024-01-10 14:12:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:14 | ERROR | stderr | response.begin() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:14 | ERROR | stderr | raise value +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:14 | ERROR | stderr | +2024-01-10 14:12:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:14 | ERROR | stderr | self.flush() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:14 | ERROR | stderr | Call stack: +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:14 | ERROR | stderr | self.run() +2024-01-10 14:12:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:14 | ERROR | stderr | Arguments: () +2024-01-10 14:12:20 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:20 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:20 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:20 | ERROR | stderr | response.begin() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:20 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:20 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:20 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:20 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:20 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:20 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:20 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:20 | ERROR | stderr | raise value +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:20 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:20 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:20 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:20 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:20 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:20 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:20 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:20 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:20 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:20 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:20 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:20 | ERROR | stderr | +2024-01-10 14:12:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:20 | ERROR | stderr | self.flush() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:20 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:20 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:20 | ERROR | stderr | Call stack: +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:20 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:20 | ERROR | stderr | self.run() +2024-01-10 14:12:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:20 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:20 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:20 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:20 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:20 | ERROR | stderr | Arguments: () +2024-01-10 14:12:24 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:24 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:24 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:24 | ERROR | stderr | response.begin() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:24 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:24 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:24 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:24 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:24 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:24 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:24 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:24 | ERROR | stderr | raise value +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:24 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:24 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:24 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:24 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:24 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:24 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:24 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:24 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:24 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:24 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:24 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:24 | ERROR | stderr | +2024-01-10 14:12:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:24 | ERROR | stderr | self.flush() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:24 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:24 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:24 | ERROR | stderr | Call stack: +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:24 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:24 | ERROR | stderr | self.run() +2024-01-10 14:12:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:24 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:24 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:24 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:24 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:24 | ERROR | stderr | Arguments: () +2024-01-10 14:12:30 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:30 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:30 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:30 | ERROR | stderr | response.begin() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:30 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:30 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:30 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:30 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:30 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:30 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:30 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:30 | ERROR | stderr | raise value +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:30 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:30 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:30 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:30 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:30 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:30 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:30 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:30 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:30 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:30 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:30 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:30 | ERROR | stderr | +2024-01-10 14:12:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:30 | ERROR | stderr | self.flush() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:30 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:30 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:30 | ERROR | stderr | Call stack: +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:30 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:30 | ERROR | stderr | self.run() +2024-01-10 14:12:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:30 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:30 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:30 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:30 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:30 | ERROR | stderr | Arguments: () +2024-01-10 14:12:34 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:34 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:34 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:34 | ERROR | stderr | response.begin() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:34 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:34 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:34 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:34 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:34 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:34 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:34 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:34 | ERROR | stderr | raise value +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:34 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:34 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:34 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:34 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:34 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:34 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:34 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:34 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:34 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:34 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:34 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:34 | ERROR | stderr | +2024-01-10 14:12:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:34 | ERROR | stderr | self.flush() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:34 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:34 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:34 | ERROR | stderr | Call stack: +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:34 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:34 | ERROR | stderr | self.run() +2024-01-10 14:12:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:34 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:34 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:34 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:34 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:34 | ERROR | stderr | Arguments: () +2024-01-10 14:12:40 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:40 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:40 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:40 | ERROR | stderr | response.begin() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:40 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:40 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:40 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:40 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:40 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:40 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:40 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:40 | ERROR | stderr | raise value +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:40 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:40 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:40 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:40 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:40 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:40 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:40 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:40 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:40 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:40 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:40 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:40 | ERROR | stderr | +2024-01-10 14:12:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:40 | ERROR | stderr | self.flush() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:40 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:40 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:40 | ERROR | stderr | Call stack: +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:40 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:40 | ERROR | stderr | self.run() +2024-01-10 14:12:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:40 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:40 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:40 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:40 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:40 | ERROR | stderr | Arguments: () +2024-01-10 14:12:44 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:44 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:44 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:44 | ERROR | stderr | response.begin() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:44 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:44 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:44 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:44 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:44 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:44 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:44 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:44 | ERROR | stderr | raise value +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:44 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:44 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:44 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:44 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:44 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:44 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:44 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:44 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:44 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:44 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:44 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:44 | ERROR | stderr | +2024-01-10 14:12:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:44 | ERROR | stderr | self.flush() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:44 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:44 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:44 | ERROR | stderr | Call stack: +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:44 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:44 | ERROR | stderr | self.run() +2024-01-10 14:12:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:44 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:44 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:44 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:44 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:44 | ERROR | stderr | Arguments: () +2024-01-10 14:12:50 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:50 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:50 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:50 | ERROR | stderr | response.begin() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:50 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:50 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:50 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:50 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:50 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:50 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:50 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:50 | ERROR | stderr | raise value +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:50 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:50 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:50 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:50 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:50 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:50 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:50 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:50 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:50 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:50 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:50 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:50 | ERROR | stderr | +2024-01-10 14:12:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:50 | ERROR | stderr | self.flush() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:50 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:50 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:50 | ERROR | stderr | Call stack: +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:50 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:50 | ERROR | stderr | self.run() +2024-01-10 14:12:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:50 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:50 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:50 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:50 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:50 | ERROR | stderr | Arguments: () +2024-01-10 14:12:54 | ERROR | stderr | --- Logging error --- +2024-01-10 14:12:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:12:54 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:12:54 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:12:54 | ERROR | stderr | response.begin() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:12:54 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:12:54 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:12:54 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:12:54 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:12:54 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:12:54 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:12:54 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:12:54 | ERROR | stderr | raise value +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:12:54 | ERROR | stderr | response = self._make_request( +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:12:54 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:12:54 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:12:54 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:12:54 | ERROR | stderr | ret = requests.post( +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:12:54 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:12:54 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:12:54 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:12:54 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:12:54 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:12:54 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:12:54 | ERROR | stderr | +2024-01-10 14:12:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:12:54 | ERROR | stderr | self.flush() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:12:54 | ERROR | stderr | self.stream.flush() +2024-01-10 14:12:54 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:12:54 | ERROR | stderr | Call stack: +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:12:54 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:12:54 | ERROR | stderr | self.run() +2024-01-10 14:12:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:12:54 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:12:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:12:54 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:12:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:12:54 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:12:54 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:12:54 | ERROR | stderr | Arguments: () +2024-01-10 14:13:00 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:00 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:00 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:00 | ERROR | stderr | response.begin() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:00 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:00 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:00 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:00 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:00 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:00 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:00 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:00 | ERROR | stderr | raise value +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:00 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:00 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:00 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:00 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:00 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:00 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:00 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:00 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:00 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:00 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:00 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:00 | ERROR | stderr | +2024-01-10 14:13:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:00 | ERROR | stderr | self.flush() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:00 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:00 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:00 | ERROR | stderr | Call stack: +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:00 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:00 | ERROR | stderr | self.run() +2024-01-10 14:13:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:00 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:00 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:00 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:00 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:00 | ERROR | stderr | Arguments: () +2024-01-10 14:13:04 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:04 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:04 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:04 | ERROR | stderr | response.begin() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:04 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:04 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:04 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:04 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:04 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:04 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:04 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:04 | ERROR | stderr | raise value +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:04 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:04 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:04 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:04 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:04 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:04 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:04 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:04 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:04 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:04 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:04 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:04 | ERROR | stderr | +2024-01-10 14:13:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:04 | ERROR | stderr | self.flush() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:04 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:04 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:04 | ERROR | stderr | Call stack: +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:04 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:04 | ERROR | stderr | self.run() +2024-01-10 14:13:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:04 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:04 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:04 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:04 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:04 | ERROR | stderr | Arguments: () +2024-01-10 14:13:10 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:10 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:10 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:10 | ERROR | stderr | response.begin() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:10 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:10 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:10 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:10 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:10 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:10 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:10 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:10 | ERROR | stderr | raise value +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:10 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:10 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:10 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:10 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:10 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:10 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:10 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:10 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:10 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:10 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:10 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:10 | ERROR | stderr | +2024-01-10 14:13:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:10 | ERROR | stderr | self.flush() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:10 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:10 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:10 | ERROR | stderr | Call stack: +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:10 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:10 | ERROR | stderr | self.run() +2024-01-10 14:13:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:10 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:10 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:10 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:10 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:10 | ERROR | stderr | Arguments: () +2024-01-10 14:13:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:14 | ERROR | stderr | response.begin() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:14 | ERROR | stderr | raise value +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:14 | ERROR | stderr | +2024-01-10 14:13:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:14 | ERROR | stderr | self.flush() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:14 | ERROR | stderr | Call stack: +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:14 | ERROR | stderr | self.run() +2024-01-10 14:13:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:14 | ERROR | stderr | Arguments: () +2024-01-10 14:13:20 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:20 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:20 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:20 | ERROR | stderr | response.begin() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:20 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:20 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:20 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:20 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:20 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:20 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:20 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:20 | ERROR | stderr | raise value +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:20 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:20 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:20 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:20 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:20 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:20 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:20 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:20 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:20 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:20 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:20 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:20 | ERROR | stderr | +2024-01-10 14:13:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:20 | ERROR | stderr | self.flush() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:20 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:20 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:20 | ERROR | stderr | Call stack: +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:20 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:20 | ERROR | stderr | self.run() +2024-01-10 14:13:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:20 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:20 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:20 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:20 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:20 | ERROR | stderr | Arguments: () +2024-01-10 14:13:24 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:24 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:24 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:24 | ERROR | stderr | response.begin() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:24 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:24 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:24 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:24 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:24 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:24 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:24 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:24 | ERROR | stderr | raise value +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:24 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:24 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:24 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:24 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:24 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:24 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:24 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:24 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:24 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:24 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:24 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:24 | ERROR | stderr | +2024-01-10 14:13:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:24 | ERROR | stderr | self.flush() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:24 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:24 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:24 | ERROR | stderr | Call stack: +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:24 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:24 | ERROR | stderr | self.run() +2024-01-10 14:13:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:24 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:24 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:24 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:24 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:24 | ERROR | stderr | Arguments: () +2024-01-10 14:13:30 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:30 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:30 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:30 | ERROR | stderr | response.begin() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:30 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:30 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:30 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:30 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:30 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:30 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:30 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:30 | ERROR | stderr | raise value +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:30 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:30 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:30 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:30 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:30 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:30 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:30 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:30 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:30 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:30 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:30 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:30 | ERROR | stderr | +2024-01-10 14:13:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:30 | ERROR | stderr | self.flush() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:30 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:30 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:30 | ERROR | stderr | Call stack: +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:30 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:30 | ERROR | stderr | self.run() +2024-01-10 14:13:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:30 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:30 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:30 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:30 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:30 | ERROR | stderr | Arguments: () +2024-01-10 14:13:34 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:34 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:34 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:34 | ERROR | stderr | response.begin() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:34 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:34 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:34 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:34 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:34 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:34 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:34 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:34 | ERROR | stderr | raise value +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:34 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:34 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:34 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:34 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:34 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:34 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:34 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:34 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:34 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:34 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:34 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:34 | ERROR | stderr | +2024-01-10 14:13:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:34 | ERROR | stderr | self.flush() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:34 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:34 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:34 | ERROR | stderr | Call stack: +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:34 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:34 | ERROR | stderr | self.run() +2024-01-10 14:13:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:34 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:34 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:34 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:34 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:34 | ERROR | stderr | Arguments: () +2024-01-10 14:13:40 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:40 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:40 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:40 | ERROR | stderr | response.begin() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:40 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:40 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:40 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:40 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:40 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:40 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:40 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:40 | ERROR | stderr | raise value +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:40 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:40 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:40 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:40 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:40 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:40 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:40 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:40 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:40 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:40 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:40 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:40 | ERROR | stderr | +2024-01-10 14:13:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:40 | ERROR | stderr | self.flush() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:40 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:40 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:40 | ERROR | stderr | Call stack: +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:40 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:40 | ERROR | stderr | self.run() +2024-01-10 14:13:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:40 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:40 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:40 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:40 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:40 | ERROR | stderr | Arguments: () +2024-01-10 14:13:44 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:44 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:44 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:44 | ERROR | stderr | response.begin() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:44 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:44 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:44 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:44 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:44 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:44 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:44 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:44 | ERROR | stderr | raise value +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:44 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:44 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:44 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:44 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:44 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:44 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:44 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:44 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:44 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:44 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:44 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:44 | ERROR | stderr | +2024-01-10 14:13:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:44 | ERROR | stderr | self.flush() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:44 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:44 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:44 | ERROR | stderr | Call stack: +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:44 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:44 | ERROR | stderr | self.run() +2024-01-10 14:13:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:44 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:44 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:44 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:44 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:44 | ERROR | stderr | Arguments: () +2024-01-10 14:13:50 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:50 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:50 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:50 | ERROR | stderr | response.begin() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:50 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:50 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:50 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:50 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:50 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:50 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:50 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:50 | ERROR | stderr | raise value +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:50 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:50 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:50 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:50 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:50 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:50 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:50 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:50 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:50 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:50 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:50 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:50 | ERROR | stderr | +2024-01-10 14:13:50 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:50 | ERROR | stderr | self.flush() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:50 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:50 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:50 | ERROR | stderr | Call stack: +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:50 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:50 | ERROR | stderr | self.run() +2024-01-10 14:13:50 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:50 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:50 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:50 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:50 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:50 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:50 | ERROR | stderr | Arguments: () +2024-01-10 14:13:54 | ERROR | stderr | --- Logging error --- +2024-01-10 14:13:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:13:54 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:13:54 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:13:54 | ERROR | stderr | response.begin() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:13:54 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:13:54 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:13:54 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:13:54 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:13:54 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:13:54 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:13:54 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:13:54 | ERROR | stderr | raise value +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:13:54 | ERROR | stderr | response = self._make_request( +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:13:54 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:13:54 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:13:54 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:13:54 | ERROR | stderr | ret = requests.post( +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:13:54 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:13:54 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:13:54 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:13:54 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:13:54 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:13:54 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:13:54 | ERROR | stderr | +2024-01-10 14:13:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:13:54 | ERROR | stderr | self.flush() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:13:54 | ERROR | stderr | self.stream.flush() +2024-01-10 14:13:54 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:13:54 | ERROR | stderr | Call stack: +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:13:54 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:13:54 | ERROR | stderr | self.run() +2024-01-10 14:13:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:13:54 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:13:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:13:54 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:13:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:13:54 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:13:54 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:13:54 | ERROR | stderr | Arguments: () +2024-01-10 14:14:00 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:00 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:00 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:00 | ERROR | stderr | response.begin() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:00 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:00 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:00 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:00 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:00 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:00 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:00 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:00 | ERROR | stderr | raise value +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:00 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:00 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:00 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:00 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:00 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:00 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:00 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:00 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:00 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:00 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:00 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:00 | ERROR | stderr | +2024-01-10 14:14:00 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:00 | ERROR | stderr | self.flush() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:00 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:00 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:00 | ERROR | stderr | Call stack: +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:00 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:00 | ERROR | stderr | self.run() +2024-01-10 14:14:00 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:00 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:00 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:00 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:00 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:00 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:00 | ERROR | stderr | Arguments: () +2024-01-10 14:14:04 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:04 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:04 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:04 | ERROR | stderr | response.begin() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:04 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:04 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:04 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:04 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:04 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:04 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:04 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:04 | ERROR | stderr | raise value +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:04 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:04 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:04 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:04 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:04 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:04 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:04 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:04 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:04 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:04 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:04 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:04 | ERROR | stderr | +2024-01-10 14:14:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:04 | ERROR | stderr | self.flush() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:04 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:04 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:04 | ERROR | stderr | Call stack: +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:04 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:04 | ERROR | stderr | self.run() +2024-01-10 14:14:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:04 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:04 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:04 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:04 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:04 | ERROR | stderr | Arguments: () +2024-01-10 14:14:10 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:10 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:10 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:10 | ERROR | stderr | response.begin() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:10 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:10 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:10 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:10 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:10 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:10 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:10 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:10 | ERROR | stderr | raise value +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:10 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:10 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:10 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:10 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:10 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:10 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:10 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:10 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:10 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:10 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:10 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:10 | ERROR | stderr | +2024-01-10 14:14:10 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:10 | ERROR | stderr | self.flush() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:10 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:10 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:10 | ERROR | stderr | Call stack: +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:10 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:10 | ERROR | stderr | self.run() +2024-01-10 14:14:10 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:10 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:10 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:10 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:10 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:10 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:10 | ERROR | stderr | Arguments: () +2024-01-10 14:14:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:14 | ERROR | stderr | response.begin() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:14 | ERROR | stderr | raise value +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:14 | ERROR | stderr | +2024-01-10 14:14:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:14 | ERROR | stderr | self.flush() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:14 | ERROR | stderr | Call stack: +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:14 | ERROR | stderr | self.run() +2024-01-10 14:14:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:14 | ERROR | stderr | Arguments: () +2024-01-10 14:14:20 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:20 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:20 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:20 | ERROR | stderr | response.begin() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:20 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:20 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:20 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:20 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:20 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:20 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:20 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:20 | ERROR | stderr | raise value +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:20 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:20 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:20 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:20 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:20 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:20 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:20 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:20 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:20 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:20 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:20 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:20 | ERROR | stderr | +2024-01-10 14:14:20 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:20 | ERROR | stderr | self.flush() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:20 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:20 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:20 | ERROR | stderr | Call stack: +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:20 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:20 | ERROR | stderr | self.run() +2024-01-10 14:14:20 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:20 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:20 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:20 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:20 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:20 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:20 | ERROR | stderr | Arguments: () +2024-01-10 14:14:24 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:24 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:24 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:24 | ERROR | stderr | response.begin() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:24 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:24 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:24 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:24 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:24 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:24 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:24 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:24 | ERROR | stderr | raise value +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:24 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:24 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:24 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:24 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:24 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:24 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:24 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:24 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:24 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:24 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:24 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:24 | ERROR | stderr | +2024-01-10 14:14:24 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:24 | ERROR | stderr | self.flush() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:24 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:24 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:24 | ERROR | stderr | Call stack: +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:24 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:24 | ERROR | stderr | self.run() +2024-01-10 14:14:24 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:24 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:24 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:24 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:24 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:24 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:24 | ERROR | stderr | Arguments: () +2024-01-10 14:14:30 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:30 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:30 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:30 | ERROR | stderr | response.begin() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:30 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:30 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:30 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:30 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:30 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:30 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:30 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:30 | ERROR | stderr | raise value +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:30 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:30 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:30 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:30 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:30 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:30 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:30 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:30 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:30 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:30 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:30 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:30 | ERROR | stderr | +2024-01-10 14:14:30 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:30 | ERROR | stderr | self.flush() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:30 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:30 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:30 | ERROR | stderr | Call stack: +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:30 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:30 | ERROR | stderr | self.run() +2024-01-10 14:14:30 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:30 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:30 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:30 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:30 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:30 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:30 | ERROR | stderr | Arguments: () +2024-01-10 14:14:34 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:34 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:34 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:34 | ERROR | stderr | response.begin() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:34 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:34 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:34 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:34 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:34 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:34 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:34 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:34 | ERROR | stderr | raise value +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:34 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:34 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:34 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:34 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:34 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:34 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:34 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:34 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:34 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:34 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:34 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:34 | ERROR | stderr | +2024-01-10 14:14:34 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:34 | ERROR | stderr | self.flush() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:34 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:34 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:34 | ERROR | stderr | Call stack: +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:34 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:34 | ERROR | stderr | self.run() +2024-01-10 14:14:34 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:34 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:34 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:34 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:34 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:34 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:34 | ERROR | stderr | Arguments: () +2024-01-10 14:14:40 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:40 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:40 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:40 | ERROR | stderr | response.begin() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:40 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:40 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:40 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:40 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:40 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:40 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:40 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:40 | ERROR | stderr | raise value +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:40 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:40 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:40 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:40 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:40 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:40 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:40 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:40 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:40 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:40 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:40 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:40 | ERROR | stderr | +2024-01-10 14:14:40 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:40 | ERROR | stderr | self.flush() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:40 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:40 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:40 | ERROR | stderr | Call stack: +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:40 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:40 | ERROR | stderr | self.run() +2024-01-10 14:14:40 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:40 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:40 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:40 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:40 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:40 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:41 | ERROR | stderr | Arguments: () +2024-01-10 14:14:44 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:44 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:44 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:44 | ERROR | stderr | response.begin() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:44 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:44 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:44 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:44 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:44 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:44 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:44 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:44 | ERROR | stderr | raise value +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:44 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:44 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:44 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:44 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:44 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:44 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:44 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:44 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:44 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:44 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:44 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:44 | ERROR | stderr | +2024-01-10 14:14:44 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:44 | ERROR | stderr | self.flush() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:44 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:44 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:44 | ERROR | stderr | Call stack: +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:44 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:44 | ERROR | stderr | self.run() +2024-01-10 14:14:44 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:44 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:44 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:44 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:44 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:44 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:44 | ERROR | stderr | Arguments: () +2024-01-10 14:14:51 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:51 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:51 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:51 | ERROR | stderr | response.begin() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:51 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:51 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:51 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:51 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:51 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:51 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:51 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:51 | ERROR | stderr | raise value +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:51 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:51 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:51 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:51 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:51 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:51 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:51 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:51 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:51 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:51 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:51 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:51 | ERROR | stderr | +2024-01-10 14:14:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:51 | ERROR | stderr | self.flush() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:51 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:51 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:51 | ERROR | stderr | Call stack: +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:51 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:51 | ERROR | stderr | self.run() +2024-01-10 14:14:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:51 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:51 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:51 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:51 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:51 | ERROR | stderr | Arguments: () +2024-01-10 14:14:54 | ERROR | stderr | --- Logging error --- +2024-01-10 14:14:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:14:54 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:14:54 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:14:54 | ERROR | stderr | response.begin() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:14:54 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:14:54 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:14:54 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:14:54 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:14:54 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:14:54 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:14:54 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:14:54 | ERROR | stderr | raise value +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:14:54 | ERROR | stderr | response = self._make_request( +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:14:54 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:14:54 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:14:54 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:14:54 | ERROR | stderr | ret = requests.post( +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:14:54 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:14:54 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:14:54 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:14:54 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:14:54 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:14:54 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:14:54 | ERROR | stderr | +2024-01-10 14:14:54 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:14:54 | ERROR | stderr | self.flush() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:14:54 | ERROR | stderr | self.stream.flush() +2024-01-10 14:14:54 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:14:54 | ERROR | stderr | Call stack: +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:14:54 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:14:54 | ERROR | stderr | self.run() +2024-01-10 14:14:54 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:14:54 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:14:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:14:54 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:14:54 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:14:54 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:14:54 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:14:54 | ERROR | stderr | Arguments: () +2024-01-10 14:15:01 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:01 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:01 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:01 | ERROR | stderr | response.begin() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:01 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:01 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:01 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:01 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:01 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:01 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:01 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:01 | ERROR | stderr | raise value +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:01 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:01 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:01 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:01 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:01 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:01 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:01 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:01 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:01 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:01 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:01 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:01 | ERROR | stderr | +2024-01-10 14:15:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:01 | ERROR | stderr | self.flush() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:01 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:01 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:01 | ERROR | stderr | Call stack: +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:01 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:01 | ERROR | stderr | self.run() +2024-01-10 14:15:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:01 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:01 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:01 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:01 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:01 | ERROR | stderr | Arguments: () +2024-01-10 14:15:04 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:04 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:04 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:04 | ERROR | stderr | response.begin() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:04 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:04 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:04 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:04 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:04 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:04 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:04 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:04 | ERROR | stderr | raise value +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:04 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:04 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:04 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:04 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:04 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:04 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:04 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:04 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:04 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:04 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:04 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:04 | ERROR | stderr | +2024-01-10 14:15:04 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:04 | ERROR | stderr | self.flush() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:04 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:04 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:04 | ERROR | stderr | Call stack: +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:04 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:04 | ERROR | stderr | self.run() +2024-01-10 14:15:04 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:04 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:04 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:04 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:04 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:04 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:04 | ERROR | stderr | Arguments: () +2024-01-10 14:15:11 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:11 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:11 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:11 | ERROR | stderr | response.begin() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:11 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:11 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:11 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:11 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:11 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:11 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:11 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:11 | ERROR | stderr | raise value +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:11 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:11 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:11 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:11 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:11 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:11 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:11 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:11 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:11 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:11 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:11 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:11 | ERROR | stderr | +2024-01-10 14:15:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:11 | ERROR | stderr | self.flush() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:11 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:11 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:11 | ERROR | stderr | Call stack: +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:11 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:11 | ERROR | stderr | self.run() +2024-01-10 14:15:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:11 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:11 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:11 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:11 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:11 | ERROR | stderr | Arguments: () +2024-01-10 14:15:14 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:14 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:14 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:14 | ERROR | stderr | response.begin() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:14 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:14 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:14 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:14 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:14 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:14 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:14 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:14 | ERROR | stderr | raise value +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:14 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:14 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:14 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:14 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:14 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:14 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:14 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:14 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:14 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:14 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:14 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:14 | ERROR | stderr | +2024-01-10 14:15:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:14 | ERROR | stderr | self.flush() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:14 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:14 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:14 | ERROR | stderr | Call stack: +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:14 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:14 | ERROR | stderr | self.run() +2024-01-10 14:15:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:14 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:14 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:14 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:14 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:14 | ERROR | stderr | Arguments: () +2024-01-10 14:15:21 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:21 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:21 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:21 | ERROR | stderr | response.begin() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:21 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:21 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:21 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:21 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:21 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:21 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:21 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:21 | ERROR | stderr | raise value +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:21 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:21 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:21 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:21 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:21 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:21 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:21 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:21 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:21 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:21 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:21 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:21 | ERROR | stderr | +2024-01-10 14:15:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:21 | ERROR | stderr | self.flush() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:21 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:21 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:21 | ERROR | stderr | Call stack: +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:21 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:21 | ERROR | stderr | self.run() +2024-01-10 14:15:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:21 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:21 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:21 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:21 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:21 | ERROR | stderr | Arguments: () +2024-01-10 14:15:25 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:25 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:25 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:25 | ERROR | stderr | response.begin() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:25 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:25 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:25 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:25 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:25 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:25 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:25 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:25 | ERROR | stderr | raise value +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:25 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:25 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:25 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:25 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:25 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:25 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:25 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:25 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:25 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:25 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:25 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:25 | ERROR | stderr | +2024-01-10 14:15:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:25 | ERROR | stderr | self.flush() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:25 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:25 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:25 | ERROR | stderr | Call stack: +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:25 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:25 | ERROR | stderr | self.run() +2024-01-10 14:15:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:25 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:25 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:25 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:25 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:25 | ERROR | stderr | Arguments: () +2024-01-10 14:15:31 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:31 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:31 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:31 | ERROR | stderr | response.begin() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:31 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:31 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:31 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:31 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:31 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:31 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:31 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:31 | ERROR | stderr | raise value +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:31 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:31 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:31 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:31 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:31 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:31 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:31 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:31 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:31 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:31 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:31 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:31 | ERROR | stderr | +2024-01-10 14:15:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:31 | ERROR | stderr | self.flush() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:31 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:31 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:31 | ERROR | stderr | Call stack: +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:31 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:31 | ERROR | stderr | self.run() +2024-01-10 14:15:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:31 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:31 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:31 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:31 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:31 | ERROR | stderr | Arguments: () +2024-01-10 14:15:35 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:35 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:35 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:35 | ERROR | stderr | response.begin() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:35 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:35 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:35 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:35 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:35 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:35 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:35 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:35 | ERROR | stderr | raise value +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:35 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:35 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:35 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:35 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:35 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:35 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:35 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:35 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:35 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:35 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:35 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:35 | ERROR | stderr | +2024-01-10 14:15:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:35 | ERROR | stderr | self.flush() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:35 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:35 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:35 | ERROR | stderr | Call stack: +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:35 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:35 | ERROR | stderr | self.run() +2024-01-10 14:15:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:35 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:35 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:35 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:35 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:35 | ERROR | stderr | Arguments: () +2024-01-10 14:15:41 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:41 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:41 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:41 | ERROR | stderr | response.begin() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:41 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:41 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:41 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:41 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:41 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:41 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:41 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:41 | ERROR | stderr | raise value +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:41 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:41 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:41 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:41 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:41 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:41 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:41 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:41 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:41 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:41 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:41 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:41 | ERROR | stderr | +2024-01-10 14:15:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:41 | ERROR | stderr | self.flush() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:41 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:41 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:41 | ERROR | stderr | Call stack: +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:41 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:41 | ERROR | stderr | self.run() +2024-01-10 14:15:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:41 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:41 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:41 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:41 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:41 | ERROR | stderr | Arguments: () +2024-01-10 14:15:45 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:45 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:45 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:45 | ERROR | stderr | response.begin() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:45 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:45 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:45 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:45 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:45 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:45 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:45 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:45 | ERROR | stderr | raise value +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:45 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:45 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:45 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:45 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:45 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:45 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:45 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:45 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:45 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:45 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:45 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:45 | ERROR | stderr | +2024-01-10 14:15:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:45 | ERROR | stderr | self.flush() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:45 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:45 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:45 | ERROR | stderr | Call stack: +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:45 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:45 | ERROR | stderr | self.run() +2024-01-10 14:15:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:45 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:45 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:45 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:45 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:45 | ERROR | stderr | Arguments: () +2024-01-10 14:15:51 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:51 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:51 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:51 | ERROR | stderr | response.begin() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:51 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:51 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:51 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:51 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:51 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:51 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:51 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:51 | ERROR | stderr | raise value +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:51 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:51 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:51 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:51 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:51 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:51 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:51 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:51 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:51 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:51 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:51 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:51 | ERROR | stderr | +2024-01-10 14:15:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:51 | ERROR | stderr | self.flush() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:51 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:51 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:51 | ERROR | stderr | Call stack: +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:51 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:51 | ERROR | stderr | self.run() +2024-01-10 14:15:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:51 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:51 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:51 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:51 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:51 | ERROR | stderr | Arguments: () +2024-01-10 14:15:55 | ERROR | stderr | --- Logging error --- +2024-01-10 14:15:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:15:55 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:15:55 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:15:55 | ERROR | stderr | response.begin() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:15:55 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:15:55 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:15:55 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:15:55 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:15:55 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:15:55 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:15:55 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:15:55 | ERROR | stderr | raise value +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:15:55 | ERROR | stderr | response = self._make_request( +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:15:55 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:15:55 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:15:55 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:15:55 | ERROR | stderr | ret = requests.post( +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:15:55 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:15:55 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:15:55 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:15:55 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:15:55 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:15:55 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:15:55 | ERROR | stderr | +2024-01-10 14:15:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:15:55 | ERROR | stderr | self.flush() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:15:55 | ERROR | stderr | self.stream.flush() +2024-01-10 14:15:55 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:15:55 | ERROR | stderr | Call stack: +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:15:55 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:15:55 | ERROR | stderr | self.run() +2024-01-10 14:15:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:15:55 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:15:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:15:55 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:15:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:15:55 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:15:55 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:15:55 | ERROR | stderr | Arguments: () +2024-01-10 14:16:01 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:01 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:01 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:01 | ERROR | stderr | response.begin() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:01 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:01 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:01 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:01 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:01 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:01 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:01 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:01 | ERROR | stderr | raise value +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:01 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:01 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:01 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:01 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:01 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:01 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:01 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:01 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:01 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:01 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:01 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:01 | ERROR | stderr | +2024-01-10 14:16:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:01 | ERROR | stderr | self.flush() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:01 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:01 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:01 | ERROR | stderr | Call stack: +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:01 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:01 | ERROR | stderr | self.run() +2024-01-10 14:16:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:01 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:01 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:01 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:01 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:01 | ERROR | stderr | Arguments: () +2024-01-10 14:16:05 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:05 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:05 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:05 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:05 | ERROR | stderr | response.begin() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:05 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:05 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:05 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:05 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:05 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:05 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:05 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:05 | ERROR | stderr | raise value +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:05 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:05 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:05 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:05 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:05 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:05 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:05 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:05 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:05 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:05 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:05 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:05 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:05 | ERROR | stderr | +2024-01-10 14:16:05 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:05 | ERROR | stderr | self.flush() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:05 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:05 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:05 | ERROR | stderr | Call stack: +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:05 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:05 | ERROR | stderr | self.run() +2024-01-10 14:16:05 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:05 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:05 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:05 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:05 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:05 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:05 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:05 | ERROR | stderr | Arguments: () +2024-01-10 14:16:11 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:11 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:11 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:11 | ERROR | stderr | response.begin() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:11 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:11 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:11 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:11 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:11 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:11 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:11 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:11 | ERROR | stderr | raise value +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:11 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:11 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:11 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:11 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:11 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:11 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:11 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:11 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:11 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:11 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:11 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:11 | ERROR | stderr | +2024-01-10 14:16:11 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:11 | ERROR | stderr | self.flush() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:11 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:11 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:11 | ERROR | stderr | Call stack: +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:11 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:11 | ERROR | stderr | self.run() +2024-01-10 14:16:11 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:11 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:11 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:11 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:11 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:11 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:11 | ERROR | stderr | Arguments: () +2024-01-10 14:16:15 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:15 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:15 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:15 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:15 | ERROR | stderr | response.begin() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:15 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:15 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:15 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:15 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:15 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:15 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:15 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:15 | ERROR | stderr | raise value +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:15 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:15 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:15 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:15 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:15 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:15 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:15 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:15 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:15 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:15 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:15 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:15 | ERROR | stderr | +2024-01-10 14:16:15 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:15 | ERROR | stderr | self.flush() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:15 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:15 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:15 | ERROR | stderr | Call stack: +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:15 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:15 | ERROR | stderr | self.run() +2024-01-10 14:16:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:15 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:15 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:15 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:15 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:15 | ERROR | stderr | Arguments: () +2024-01-10 14:16:21 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:21 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:21 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:21 | ERROR | stderr | response.begin() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:21 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:21 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:21 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:21 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:21 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:21 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:21 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:21 | ERROR | stderr | raise value +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:21 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:21 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:21 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:21 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:21 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:21 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:21 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:21 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:21 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:21 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:21 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:21 | ERROR | stderr | +2024-01-10 14:16:21 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:21 | ERROR | stderr | self.flush() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:21 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:21 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:21 | ERROR | stderr | Call stack: +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:21 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:21 | ERROR | stderr | self.run() +2024-01-10 14:16:21 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:21 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:21 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:21 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:21 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:21 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:21 | ERROR | stderr | Arguments: () +2024-01-10 14:16:25 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:25 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:25 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:25 | ERROR | stderr | response.begin() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:25 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:25 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:25 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:25 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:25 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:25 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:25 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:25 | ERROR | stderr | raise value +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:25 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:25 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:25 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:25 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:25 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:25 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:25 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:25 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:25 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:25 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:25 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:25 | ERROR | stderr | +2024-01-10 14:16:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:25 | ERROR | stderr | self.flush() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:25 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:25 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:25 | ERROR | stderr | Call stack: +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:25 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:25 | ERROR | stderr | self.run() +2024-01-10 14:16:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:25 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:25 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:25 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:25 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:25 | ERROR | stderr | Arguments: () +2024-01-10 14:16:31 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:31 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:31 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:31 | ERROR | stderr | response.begin() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:31 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:31 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:31 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:31 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:31 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:31 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:31 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:31 | ERROR | stderr | raise value +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:31 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:31 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:31 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:31 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:31 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:31 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:31 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:31 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:31 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:31 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:31 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:31 | ERROR | stderr | +2024-01-10 14:16:31 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:31 | ERROR | stderr | self.flush() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:31 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:31 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:31 | ERROR | stderr | Call stack: +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:31 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:31 | ERROR | stderr | self.run() +2024-01-10 14:16:31 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:31 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:31 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:31 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:31 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:31 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:31 | ERROR | stderr | Arguments: () +2024-01-10 14:16:35 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:35 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:35 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:35 | ERROR | stderr | response.begin() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:35 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:35 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:35 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:35 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:35 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:35 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:35 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:35 | ERROR | stderr | raise value +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:35 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:35 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:35 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:35 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:35 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:35 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:35 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:35 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:35 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:35 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:35 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:35 | ERROR | stderr | +2024-01-10 14:16:35 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:35 | ERROR | stderr | self.flush() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:35 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:35 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:35 | ERROR | stderr | Call stack: +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:35 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:35 | ERROR | stderr | self.run() +2024-01-10 14:16:35 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:35 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:35 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:35 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:35 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:35 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:35 | ERROR | stderr | Arguments: () +2024-01-10 14:16:41 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:41 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:41 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:41 | ERROR | stderr | response.begin() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:41 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:41 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:41 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:41 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:41 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:41 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:41 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:41 | ERROR | stderr | raise value +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:41 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:41 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:41 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:41 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:41 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:41 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:41 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:41 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:41 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:41 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:41 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:41 | ERROR | stderr | +2024-01-10 14:16:41 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:41 | ERROR | stderr | self.flush() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:41 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:41 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:41 | ERROR | stderr | Call stack: +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:41 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:41 | ERROR | stderr | self.run() +2024-01-10 14:16:41 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:41 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:41 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:41 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:41 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:41 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:41 | ERROR | stderr | Arguments: () +2024-01-10 14:16:45 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:45 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:45 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:45 | ERROR | stderr | response.begin() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:45 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:45 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:45 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:45 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:45 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:45 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:45 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:45 | ERROR | stderr | raise value +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:45 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:45 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:45 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:45 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:45 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:45 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:45 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:45 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:45 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:45 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:45 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:45 | ERROR | stderr | +2024-01-10 14:16:45 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:45 | ERROR | stderr | self.flush() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:45 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:45 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:45 | ERROR | stderr | Call stack: +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:45 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:45 | ERROR | stderr | self.run() +2024-01-10 14:16:45 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:45 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:45 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:45 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:45 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:45 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:45 | ERROR | stderr | Arguments: () +2024-01-10 14:16:51 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:51 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:51 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:51 | ERROR | stderr | response.begin() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:51 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:51 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:51 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:51 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:51 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:51 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:51 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:51 | ERROR | stderr | raise value +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:51 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:51 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:51 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:51 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:51 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:51 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:51 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:51 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:51 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:51 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:51 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:51 | ERROR | stderr | +2024-01-10 14:16:51 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:51 | ERROR | stderr | self.flush() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:51 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:51 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:51 | ERROR | stderr | Call stack: +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:51 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:51 | ERROR | stderr | self.run() +2024-01-10 14:16:51 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:51 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:51 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:51 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:51 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:51 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:51 | ERROR | stderr | Arguments: () +2024-01-10 14:16:55 | ERROR | stderr | --- Logging error --- +2024-01-10 14:16:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:16:55 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:16:55 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:16:55 | ERROR | stderr | response.begin() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:16:55 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:16:55 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:16:55 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:16:55 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:16:55 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:16:55 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:16:55 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:16:55 | ERROR | stderr | raise value +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:16:55 | ERROR | stderr | response = self._make_request( +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:16:55 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:16:55 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:16:55 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:16:55 | ERROR | stderr | ret = requests.post( +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:16:55 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:16:55 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:16:55 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:16:55 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:16:55 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:16:55 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:16:55 | ERROR | stderr | +2024-01-10 14:16:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:16:55 | ERROR | stderr | self.flush() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:16:55 | ERROR | stderr | self.stream.flush() +2024-01-10 14:16:55 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:16:55 | ERROR | stderr | Call stack: +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:16:55 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:16:55 | ERROR | stderr | self.run() +2024-01-10 14:16:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:16:55 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:16:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:16:55 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:16:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:16:55 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:16:55 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:16:55 | ERROR | stderr | Arguments: () +2024-01-10 14:17:01 | ERROR | stderr | --- Logging error --- +2024-01-10 14:17:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:17:01 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:17:01 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:17:01 | ERROR | stderr | response.begin() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:17:01 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:17:01 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:17:01 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:17:01 | ERROR | stderr | socket.timeout: timed out +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:17:01 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-10 14:17:01 | ERROR | stderr | retries = retries.increment( +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment +2024-01-10 14:17:01 | ERROR | stderr | raise reraise(type(error), error, _stacktrace) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/util.py", line 39, in reraise +2024-01-10 14:17:01 | ERROR | stderr | raise value +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:17:01 | ERROR | stderr | response = self._make_request( +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 538, in _make_request +2024-01-10 14:17:01 | ERROR | stderr | self._raise_timeout(err=e, url=url, timeout_value=read_timeout) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout +2024-01-10 14:17:01 | ERROR | stderr | raise ReadTimeoutError( +2024-01-10 14:17:01 | ERROR | stderr | urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:17:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 111, in send_heart_beat +2024-01-10 14:17:01 | ERROR | stderr | ret = requests.post( +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:17:01 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:17:01 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:17:01 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:17:01 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 532, in send +2024-01-10 14:17:01 | ERROR | stderr | raise ReadTimeout(e, request=request) +2024-01-10 14:17:01 | ERROR | stderr | requests.exceptions.ReadTimeout: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5) +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 14:17:01 | ERROR | stderr | +2024-01-10 14:17:01 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1087, in emit +2024-01-10 14:17:01 | ERROR | stderr | self.flush() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/logging/__init__.py", line 1067, in flush +2024-01-10 14:17:01 | ERROR | stderr | self.stream.flush() +2024-01-10 14:17:01 | ERROR | stderr | OSError: [Errno 122] Disk quota exceeded +2024-01-10 14:17:01 | ERROR | stderr | Call stack: +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 937, in _bootstrap +2024-01-10 14:17:01 | ERROR | stderr | self._bootstrap_inner() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 980, in _bootstrap_inner +2024-01-10 14:17:01 | ERROR | stderr | self.run() +2024-01-10 14:17:01 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/threading.py", line 917, in run +2024-01-10 14:17:01 | ERROR | stderr | self._target(*self._args, **self._kwargs) +2024-01-10 14:17:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 24, in heart_beat_worker +2024-01-10 14:17:01 | ERROR | stderr | obj.send_heart_beat() +2024-01-10 14:17:01 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 122, in send_heart_beat +2024-01-10 14:17:01 | ERROR | stderr | logger.error(f"heart beat error: {e}") +2024-01-10 14:17:01 | ERROR | stderr | Message: "heart beat error: HTTPConnectionPool(host='172.17.8.132', port=21001): Read timed out. (read timeout=5)" +2024-01-10 14:17:01 | ERROR | stderr | Arguments: () +2024-01-10 14:17:11 | ERROR | stderr | 0%| | 0.00/1.36M [00:00 +2024-01-10 14:26:57 | ERROR | stderr | models = get_model_list( +2024-01-10 14:26:57 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 169, in get_model_list +2024-01-10 14:26:57 | ERROR | stderr | ret = requests.post(controller_url + "/refresh_all_workers") +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:26:57 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:26:57 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:26:57 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:26:57 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:26:57 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:26:57 | ERROR | stderr | response = self._make_request( +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:26:57 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:26:57 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:26:57 | ERROR | stderr | response.begin() +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:26:57 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:26:57 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:26:57 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:26:57 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:26:57 | ERROR | stderr | KeyboardInterrupt +2024-01-10 14:28:53 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 14:28:53 | INFO | stdout | +2024-01-10 14:28:53 | ERROR | stderr | /fs01/home/tianleli/ImagenHub/src/imagen_hub/pipelines/blended_diffusion/blended_diffusion_src/CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended +2024-01-10 14:28:53 | ERROR | stderr | warnings.warn("PyTorch version 1.7.1 or higher is recommended") +2024-01-10 14:28:54 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 14:28:54 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 14:28:54 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 14:31:23 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 14:31:23 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 14:31:23 | ERROR | stderr | exec(code, run_globals) +2024-01-10 14:31:23 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server_multi.py", line 256, in +2024-01-10 14:31:23 | ERROR | stderr | models = get_model_list( +2024-01-10 14:31:23 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 169, in get_model_list +2024-01-10 14:31:23 | ERROR | stderr | ret = requests.post(controller_url + "/refresh_all_workers") +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-10 14:31:23 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-10 14:31:23 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-10 14:31:23 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-10 14:31:23 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-10 14:31:23 | ERROR | stderr | resp = conn.urlopen( +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen +2024-01-10 14:31:23 | ERROR | stderr | response = self._make_request( +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request +2024-01-10 14:31:23 | ERROR | stderr | response = conn.getresponse() +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connection.py", line 461, in getresponse +2024-01-10 14:31:23 | ERROR | stderr | httplib_response = super().getresponse() +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 1377, in getresponse +2024-01-10 14:31:23 | ERROR | stderr | response.begin() +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 320, in begin +2024-01-10 14:31:23 | ERROR | stderr | version, status, reason = self._read_status() +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/http/client.py", line 281, in _read_status +2024-01-10 14:31:23 | ERROR | stderr | line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") +2024-01-10 14:31:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/socket.py", line 704, in readinto +2024-01-10 14:31:23 | ERROR | stderr | return self._sock.recv_into(b) +2024-01-10 14:31:23 | ERROR | stderr | KeyboardInterrupt +2024-01-10 14:40:42 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 14:40:42 | INFO | stdout | +2024-01-10 14:40:42 | ERROR | stderr | /fs01/home/tianleli/ImagenHub/src/imagen_hub/pipelines/blended_diffusion/blended_diffusion_src/CLIP/clip/clip.py:23: UserWarning: PyTorch version 1.7.1 or higher is recommended +2024-01-10 14:40:42 | ERROR | stderr | warnings.warn("PyTorch version 1.7.1 or higher is recommended") +2024-01-10 14:40:48 | ERROR | stderr | Loading pipeline components...: 0%| | 0/6 [00:00 +2024-01-10 14:42:19 | ERROR | stderr | args, worker = create_model_worker() +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 362, in create_model_worker +2024-01-10 14:42:19 | ERROR | stderr | worker = ModelWorker( +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 86, in __init__ +2024-01-10 14:42:19 | ERROR | stderr | self.model, self.tokenizer = load_model( +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 332, in load_model +2024-01-10 14:42:19 | ERROR | stderr | model, tokenizer = adapter.load_model(model_path, kwargs) +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 2043, in load_model +2024-01-10 14:42:19 | ERROR | stderr | model = imagen_hub.load(model_name) +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/__init__.py", line 86, in get_model +2024-01-10 14:42:19 | ERROR | stderr | model_instance = model_class() +2024-01-10 14:42:19 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/sd.py", line 115, in __init__ +2024-01-10 14:42:19 | ERROR | stderr | self.pipe = DiffusionPipeline.from_pretrained( +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1090, in from_pretrained +2024-01-10 14:42:19 | ERROR | stderr | cached_folder = cls.download( +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1856, in download +2024-01-10 14:42:19 | ERROR | stderr | cached_folder = snapshot_download( +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:42:19 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 308, in snapshot_download +2024-01-10 14:42:19 | ERROR | stderr | thread_map( +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map +2024-01-10 14:42:19 | ERROR | stderr | return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map +2024-01-10 14:42:19 | ERROR | stderr | return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs)) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/std.py", line 1182, in __iter__ +2024-01-10 14:42:19 | ERROR | stderr | for obj in iterable: +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 609, in result_iterator +2024-01-10 14:42:19 | ERROR | stderr | yield fs.pop().result() +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 446, in result +2024-01-10 14:42:19 | ERROR | stderr | return self.__get_result() +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result +2024-01-10 14:42:19 | ERROR | stderr | raise self._exception +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/thread.py", line 58, in run +2024-01-10 14:42:19 | ERROR | stderr | result = self.fn(*self.args, **self.kwargs) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 283, in _inner_hf_hub_download +2024-01-10 14:42:19 | ERROR | stderr | return hf_hub_download( +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:42:19 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1468, in hf_hub_download +2024-01-10 14:42:19 | ERROR | stderr | _chmod_and_replace(temp_file.name, blob_path) +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1683, in _chmod_and_replace +2024-01-10 14:42:19 | ERROR | stderr | tmp_file.unlink() +2024-01-10 14:42:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/pathlib.py", line 1354, in unlink +2024-01-10 14:42:19 | ERROR | stderr | self._accessor.unlink(self) +2024-01-10 14:42:19 | ERROR | stderr | FileNotFoundError: [Errno 2] No such file or directory: '/h/tianleli/.cache/huggingface/hub/models--SimianLuo--LCM_Dreamshaper_v7/tmp_5afcf45d-0951-4807-8adf-ad4d3c03ba52' +2024-01-10 14:42:23 | ERROR | stderr | model_index.json: 0%| | 0.00/400 [00:00 +2024-01-10 14:43:13 | ERROR | stderr | args, worker = create_model_worker() +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 362, in create_model_worker +2024-01-10 14:43:13 | ERROR | stderr | worker = ModelWorker( +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 86, in __init__ +2024-01-10 14:43:13 | ERROR | stderr | self.model, self.tokenizer = load_model( +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 332, in load_model +2024-01-10 14:43:13 | ERROR | stderr | model, tokenizer = adapter.load_model(model_path, kwargs) +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 2043, in load_model +2024-01-10 14:43:13 | ERROR | stderr | model = imagen_hub.load(model_name) +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/__init__.py", line 86, in get_model +2024-01-10 14:43:13 | ERROR | stderr | model_instance = model_class() +2024-01-10 14:43:13 | ERROR | stderr | File "/fs01/home/tianleli/ImagenHub/src/imagen_hub/infermodels/pixart_alpha.py", line 19, in __init__ +2024-01-10 14:43:13 | ERROR | stderr | self.pipe = PixArtAlphaPipeline.from_pretrained( +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1090, in from_pretrained +2024-01-10 14:43:13 | ERROR | stderr | cached_folder = cls.download( +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/pipelines/pipeline_utils.py", line 1856, in download +2024-01-10 14:43:13 | ERROR | stderr | cached_folder = snapshot_download( +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:43:13 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 308, in snapshot_download +2024-01-10 14:43:13 | ERROR | stderr | thread_map( +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map +2024-01-10 14:43:13 | ERROR | stderr | return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map +2024-01-10 14:43:13 | ERROR | stderr | return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs)) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/tqdm/std.py", line 1182, in __iter__ +2024-01-10 14:43:13 | ERROR | stderr | for obj in iterable: +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 609, in result_iterator +2024-01-10 14:43:13 | ERROR | stderr | yield fs.pop().result() +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 446, in result +2024-01-10 14:43:13 | ERROR | stderr | return self.__get_result() +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result +2024-01-10 14:43:13 | ERROR | stderr | raise self._exception +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/concurrent/futures/thread.py", line 58, in run +2024-01-10 14:43:13 | ERROR | stderr | result = self.fn(*self.args, **self.kwargs) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/_snapshot_download.py", line 283, in _inner_hf_hub_download +2024-01-10 14:43:13 | ERROR | stderr | return hf_hub_download( +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn +2024-01-10 14:43:13 | ERROR | stderr | return fn(*args, **kwargs) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1468, in hf_hub_download +2024-01-10 14:43:13 | ERROR | stderr | _chmod_and_replace(temp_file.name, blob_path) +2024-01-10 14:43:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1681, in _chmod_and_replace +2024-01-10 14:43:13 | ERROR | stderr | os.chmod(src, stat.S_IMODE(cache_dir_mode)) +2024-01-10 14:43:13 | ERROR | stderr | FileNotFoundError: [Errno 2] No such file or directory: '/h/tianleli/.cache/huggingface/hub/tmp_mh0v77i' +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 24%|██▍ | 2.44G/9.99G [00:15<00:41, 182MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 27%|██▋ | 2.43G/9.06G [00:15<00:36, 180MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | diffusion_pytorch_model.safetensors: 97%|█████████▋| 2.37G/2.45G [00:14<00:00, 157MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 25%|██▍ | 2.46G/9.99G [00:15<00:42, 176MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 27%|██▋ | 2.46G/9.06G [00:15<00:34, 193MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | diffusion_pytorch_model.safetensors: 98%|█████████▊| 2.39G/2.45G [00:15<00:00, 161MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 27%|██▋ | 2.49G/9.06G [00:15<00:33, 195MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 25%|██▍ | 2.49G/9.99G [00:15<00:44, 169MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | diffusion_pytorch_model.safetensors: 99%|█████████▊| 2.41G/2.45G [00:15<00:00, 162MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 28%|██▊ | 2.51G/9.06G [00:15<00:33, 197MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 25%|██▌ | 2.51G/9.99G [00:15<00:45, 166MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | diffusion_pytorch_model.safetensors: 99%|█████████▉| 2.43G/2.45G [00:15<00:00, 157MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 28%|██▊ | 2.54G/9.06G [00:15<00:32, 202MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 25%|██▌ | 2.53G/9.99G [00:15<00:46, 162MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | diffusion_pytorch_model.safetensors: 100%|██████████| 2.45G/2.45G [00:15<00:00, 158MB/s] +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 28%|██▊ | 2.56G/9.06G [00:15<00:33, 193MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 26%|██▌ | 2.55G/9.99G [00:15<00:44, 169MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00002-of-00002.safetensors: 29%|██▊ | 2.59G/9.06G [00:15<00:31, 204MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:13 | ERROR | stderr | +2024-01-10 14:43:13 | ERROR | stderr | model-00001-of-00002.safetensors: 26%|██▌ | 2.58G/9.99G [00:15<00:38, 191MB/s] +2024-01-10 14:43:13 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 26%|██▌ | 2.60G/9.99G [00:16<00:38, 194MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 29%|██▉ | 2.62G/9.06G [00:16<00:30, 212MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 26%|██▋ | 2.63G/9.99G [00:16<00:34, 212MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 29%|██▉ | 2.65G/9.06G [00:16<00:30, 213MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 27%|██▋ | 2.66G/9.99G [00:16<00:33, 219MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 30%|██▉ | 2.68G/9.06G [00:16<00:29, 219MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 27%|██▋ | 2.69G/9.99G [00:16<00:32, 225MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 30%|██▉ | 2.72G/9.06G [00:16<00:28, 225MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 27%|██▋ | 2.73G/9.99G [00:16<00:31, 227MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 30%|███ | 2.75G/9.06G [00:16<00:27, 228MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 28%|██▊ | 2.76G/9.99G [00:16<00:31, 228MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 31%|███ | 2.78G/9.06G [00:16<00:27, 232MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 28%|██▊ | 2.79G/9.99G [00:16<00:31, 228MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00002-of-00002.safetensors: 31%|███ | 2.81G/9.06G [00:16<00:27, 226MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:14 | ERROR | stderr | +2024-01-10 14:43:14 | ERROR | stderr | model-00001-of-00002.safetensors: 28%|██▊ | 2.82G/9.99G [00:16<00:31, 230MB/s] +2024-01-10 14:43:14 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 31%|███▏ | 2.84G/9.06G [00:16<00:27, 229MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 29%|██▊ | 2.85G/9.99G [00:17<00:30, 230MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 32%|███▏ | 2.87G/9.06G [00:17<00:26, 230MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 29%|██▉ | 2.88G/9.99G [00:17<00:30, 232MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 32%|███▏ | 2.90G/9.06G [00:17<00:26, 231MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 29%|██▉ | 2.92G/9.99G [00:17<00:30, 231MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 32%|███▏ | 2.94G/9.06G [00:17<00:26, 232MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 29%|██▉ | 2.95G/9.99G [00:17<00:30, 230MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 33%|███▎ | 2.97G/9.06G [00:17<00:25, 235MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 30%|██▉ | 2.98G/9.99G [00:17<00:30, 227MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 33%|███▎ | 3.00G/9.06G [00:17<00:25, 235MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 30%|███ | 3.01G/9.99G [00:17<00:29, 233MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 33%|███▎ | 3.03G/9.06G [00:17<00:26, 230MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00001-of-00002.safetensors: 30%|███ | 3.04G/9.99G [00:17<00:29, 233MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | +2024-01-10 14:43:15 | ERROR | stderr | model-00002-of-00002.safetensors: 34%|███▍ | 3.06G/9.06G [00:17<00:26, 229MB/s] +2024-01-10 14:43:15 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 31%|███ | 3.07G/9.99G [00:18<00:29, 232MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 34%|███▍ | 3.09G/9.06G [00:18<00:26, 229MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 31%|███ | 3.10G/9.99G [00:18<00:29, 232MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 34%|███▍ | 3.12G/9.06G [00:18<00:25, 230MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 31%|███▏ | 3.14G/9.99G [00:18<00:30, 228MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 35%|███▍ | 3.16G/9.06G [00:18<00:25, 232MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 32%|███▏ | 3.17G/9.99G [00:18<00:30, 223MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 35%|███▌ | 3.19G/9.06G [00:18<00:25, 233MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 32%|███▏ | 3.20G/9.99G [00:18<00:30, 226MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 36%|███▌ | 3.22G/9.06G [00:18<00:25, 231MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 32%|███▏ | 3.23G/9.99G [00:18<00:29, 227MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 36%|███▌ | 3.25G/9.06G [00:18<00:25, 230MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00001-of-00002.safetensors: 33%|███▎ | 3.26G/9.99G [00:18<00:29, 228MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | +2024-01-10 14:43:16 | ERROR | stderr | model-00002-of-00002.safetensors: 36%|███▌ | 3.28G/9.06G [00:18<00:25, 230MB/s] +2024-01-10 14:43:16 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 33%|███▎ | 3.29G/9.99G [00:18<00:28, 235MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 37%|███▋ | 3.31G/9.06G [00:19<00:25, 227MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 33%|███▎ | 3.32G/9.99G [00:19<00:28, 235MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 37%|███▋ | 3.34G/9.06G [00:19<00:24, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 34%|███▎ | 3.36G/9.99G [00:19<00:28, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 37%|███▋ | 3.38G/9.06G [00:19<00:24, 229MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 34%|███▍ | 3.39G/9.99G [00:19<00:28, 234MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 38%|███▊ | 3.41G/9.06G [00:19<00:24, 229MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 34%|███▍ | 3.42G/9.99G [00:19<00:28, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 38%|███▊ | 3.44G/9.06G [00:19<00:24, 229MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 35%|███▍ | 3.45G/9.99G [00:19<00:28, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 38%|███▊ | 3.47G/9.06G [00:19<00:24, 230MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 35%|███▍ | 3.48G/9.99G [00:19<00:27, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 39%|███▊ | 3.50G/9.06G [00:19<00:23, 232MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00001-of-00002.safetensors: 35%|███▌ | 3.51G/9.99G [00:19<00:27, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | +2024-01-10 14:43:17 | ERROR | stderr | model-00002-of-00002.safetensors: 39%|███▉ | 3.53G/9.06G [00:19<00:23, 233MB/s] +2024-01-10 14:43:17 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 35%|███▌ | 3.54G/9.99G [00:20<00:27, 233MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 39%|███▉ | 3.57G/9.06G [00:20<00:23, 232MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 36%|███▌ | 3.58G/9.99G [00:20<00:28, 226MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 40%|███▉ | 3.60G/9.06G [00:20<00:23, 237MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 36%|███▌ | 3.61G/9.99G [00:20<00:28, 227MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 40%|████ | 3.63G/9.06G [00:20<00:23, 234MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 36%|███▋ | 3.64G/9.99G [00:20<00:27, 228MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 40%|████ | 3.66G/9.06G [00:20<00:23, 233MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 37%|███▋ | 3.67G/9.99G [00:20<00:27, 230MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 41%|████ | 3.69G/9.06G [00:20<00:23, 233MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 37%|███▋ | 3.70G/9.99G [00:20<00:27, 227MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 41%|████ | 3.72G/9.06G [00:20<00:22, 233MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00002-of-00002.safetensors: 41%|████▏ | 3.75G/9.06G [00:20<00:21, 243MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:18 | ERROR | stderr | +2024-01-10 14:43:18 | ERROR | stderr | model-00001-of-00002.safetensors: 37%|███▋ | 3.73G/9.99G [00:20<00:28, 221MB/s] +2024-01-10 14:43:18 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 38%|███▊ | 3.76G/9.99G [00:21<00:27, 225MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 42%|████▏ | 3.79G/9.06G [00:21<00:23, 227MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 38%|███▊ | 3.80G/9.99G [00:21<00:27, 226MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 42%|████▏ | 3.82G/9.06G [00:21<00:23, 225MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 38%|███▊ | 3.83G/9.99G [00:21<00:26, 229MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 42%|████▏ | 3.85G/9.06G [00:21<00:22, 227MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 39%|███▊ | 3.86G/9.99G [00:21<00:26, 232MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 43%|████▎ | 3.88G/9.06G [00:21<00:22, 231MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 39%|███▉ | 3.89G/9.99G [00:21<00:26, 232MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 43%|████▎ | 3.91G/9.06G [00:21<00:22, 232MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 39%|███▉ | 3.92G/9.99G [00:21<00:26, 233MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 44%|████▎ | 3.94G/9.06G [00:21<00:22, 232MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00002-of-00002.safetensors: 44%|████▍ | 3.97G/9.06G [00:21<00:20, 244MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:19 | ERROR | stderr | +2024-01-10 14:43:19 | ERROR | stderr | model-00001-of-00002.safetensors: 40%|███▉ | 3.95G/9.99G [00:21<00:27, 223MB/s] +2024-01-10 14:43:19 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 44%|████▍ | 4.01G/9.06G [00:21<00:20, 242MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 40%|███▉ | 3.98G/9.99G [00:22<00:26, 226MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 45%|████▍ | 4.04G/9.06G [00:22<00:20, 240MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 40%|████ | 4.02G/9.99G [00:22<00:26, 229MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 45%|████▍ | 4.07G/9.06G [00:22<00:20, 239MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 41%|████ | 4.05G/9.99G [00:22<00:25, 232MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 41%|████ | 4.08G/9.99G [00:22<00:24, 241MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 45%|████▌ | 4.10G/9.06G [00:22<00:21, 227MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 41%|████ | 4.11G/9.99G [00:22<00:24, 241MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 46%|████▌ | 4.13G/9.06G [00:22<00:21, 229MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 41%|████▏ | 4.14G/9.99G [00:22<00:24, 238MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 46%|████▌ | 4.16G/9.06G [00:22<00:21, 232MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 46%|████▋ | 4.19G/9.06G [00:22<00:20, 242MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 42%|████▏ | 4.17G/9.99G [00:22<00:25, 229MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00002-of-00002.safetensors: 47%|████▋ | 4.23G/9.06G [00:22<00:20, 239MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:20 | ERROR | stderr | +2024-01-10 14:43:20 | ERROR | stderr | model-00001-of-00002.safetensors: 42%|████▏ | 4.20G/9.99G [00:22<00:25, 229MB/s] +2024-01-10 14:43:20 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 47%|████▋ | 4.26G/9.06G [00:23<00:20, 236MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 42%|████▏ | 4.24G/9.99G [00:23<00:25, 228MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 47%|████▋ | 4.29G/9.06G [00:23<00:20, 235MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 43%|████▎ | 4.27G/9.99G [00:23<00:24, 229MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 48%|████▊ | 4.32G/9.06G [00:23<00:20, 233MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 43%|████▎ | 4.30G/9.99G [00:23<00:24, 230MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 48%|████▊ | 4.35G/9.06G [00:23<00:20, 234MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 43%|████▎ | 4.33G/9.99G [00:23<00:24, 231MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 44%|████▎ | 4.36G/9.99G [00:23<00:23, 240MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 48%|████▊ | 4.38G/9.06G [00:23<00:21, 222MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 44%|████▍ | 4.39G/9.99G [00:23<00:23, 240MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 49%|████▊ | 4.41G/9.06G [00:23<00:20, 225MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00001-of-00002.safetensors: 44%|████▍ | 4.42G/9.99G [00:23<00:23, 238MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | +2024-01-10 14:43:21 | ERROR | stderr | model-00002-of-00002.safetensors: 49%|████▉ | 4.45G/9.06G [00:23<00:20, 228MB/s] +2024-01-10 14:43:21 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 49%|████▉ | 4.48G/9.06G [00:23<00:19, 240MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 45%|████▍ | 4.46G/9.99G [00:24<00:24, 226MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 50%|████▉ | 4.51G/9.06G [00:24<00:19, 236MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 45%|████▍ | 4.49G/9.99G [00:24<00:24, 225MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 50%|█████ | 4.54G/9.06G [00:24<00:19, 233MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 45%|████▌ | 4.52G/9.99G [00:24<00:24, 227MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 50%|█████ | 4.57G/9.06G [00:24<00:19, 233MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 46%|████▌ | 4.55G/9.99G [00:24<00:23, 228MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 51%|█████ | 4.60G/9.06G [00:24<00:19, 233MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 46%|████▌ | 4.58G/9.99G [00:24<00:23, 229MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 51%|█████ | 4.63G/9.06G [00:24<00:18, 234MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 46%|████▌ | 4.61G/9.99G [00:24<00:23, 231MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 52%|█████▏ | 4.67G/9.06G [00:24<00:19, 230MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00001-of-00002.safetensors: 47%|████▋ | 4.65G/9.99G [00:24<00:23, 230MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | +2024-01-10 14:43:22 | ERROR | stderr | model-00002-of-00002.safetensors: 52%|█████▏ | 4.70G/9.06G [00:24<00:19, 230MB/s] +2024-01-10 14:43:22 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 47%|████▋ | 4.68G/9.99G [00:25<00:23, 229MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 52%|█████▏ | 4.73G/9.06G [00:25<00:18, 232MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 47%|████▋ | 4.71G/9.99G [00:25<00:22, 231MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 53%|█████▎ | 4.76G/9.06G [00:25<00:18, 232MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 47%|████▋ | 4.74G/9.99G [00:25<00:22, 232MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 53%|█████▎ | 4.79G/9.06G [00:25<00:18, 233MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 48%|████▊ | 4.77G/9.99G [00:25<00:22, 232MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 53%|█████▎ | 4.82G/9.06G [00:25<00:19, 222MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 48%|████▊ | 4.80G/9.99G [00:25<00:23, 222MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 54%|█████▎ | 4.85G/9.06G [00:25<00:18, 225MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 48%|████▊ | 4.83G/9.99G [00:25<00:22, 225MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 54%|█████▍ | 4.89G/9.06G [00:25<00:18, 227MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 49%|████▊ | 4.87G/9.99G [00:25<00:22, 227MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00002-of-00002.safetensors: 54%|█████▍ | 4.92G/9.06G [00:25<00:18, 229MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:23 | ERROR | stderr | +2024-01-10 14:43:23 | ERROR | stderr | model-00001-of-00002.safetensors: 49%|████▉ | 4.90G/9.99G [00:25<00:22, 229MB/s] +2024-01-10 14:43:23 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 55%|█████▍ | 4.95G/9.06G [00:26<00:17, 231MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 49%|████▉ | 4.93G/9.99G [00:26<00:21, 231MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 55%|█████▍ | 4.98G/9.06G [00:26<00:17, 229MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 50%|████▉ | 4.96G/9.99G [00:26<00:21, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 55%|█████▌ | 5.01G/9.06G [00:26<00:17, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 50%|████▉ | 4.99G/9.99G [00:26<00:21, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 56%|█████▌ | 5.04G/9.06G [00:26<00:17, 229MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 50%|█████ | 5.02G/9.99G [00:26<00:21, 229MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 56%|█████▌ | 5.08G/9.06G [00:26<00:17, 229MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 51%|█████ | 5.05G/9.99G [00:26<00:21, 229MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 56%|█████▋ | 5.11G/9.06G [00:26<00:17, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 51%|█████ | 5.09G/9.99G [00:26<00:21, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00002-of-00002.safetensors: 57%|█████▋ | 5.14G/9.06G [00:26<00:17, 230MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:24 | ERROR | stderr | +2024-01-10 14:43:24 | ERROR | stderr | model-00001-of-00002.safetensors: 51%|█████ | 5.12G/9.99G [00:26<00:21, 231MB/s] +2024-01-10 14:43:24 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 57%|█████▋ | 5.17G/9.06G [00:27<00:16, 229MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 52%|█████▏ | 5.15G/9.99G [00:27<00:21, 230MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 57%|█████▋ | 5.20G/9.06G [00:27<00:16, 231MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 52%|█████▏ | 5.18G/9.99G [00:27<00:20, 229MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 58%|█████▊ | 5.23G/9.06G [00:27<00:16, 229MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 52%|█████▏ | 5.21G/9.99G [00:27<00:20, 229MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 58%|█████▊ | 5.26G/9.06G [00:27<00:16, 230MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 52%|█████▏ | 5.24G/9.99G [00:27<00:20, 230MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 58%|█████▊ | 5.30G/9.06G [00:27<00:16, 231MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 53%|█████▎ | 5.27G/9.99G [00:27<00:20, 230MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 59%|█████▉ | 5.33G/9.06G [00:27<00:15, 234MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 53%|█████▎ | 5.31G/9.99G [00:27<00:20, 225MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 59%|█████▉ | 5.36G/9.06G [00:27<00:15, 236MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00001-of-00002.safetensors: 53%|█████▎ | 5.34G/9.99G [00:27<00:21, 215MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | +2024-01-10 14:43:25 | ERROR | stderr | model-00002-of-00002.safetensors: 59%|█████▉ | 5.39G/9.06G [00:27<00:15, 233MB/s] +2024-01-10 14:43:25 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 54%|█████▎ | 5.37G/9.99G [00:28<00:20, 226MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 60%|█████▉ | 5.42G/9.06G [00:28<00:16, 223MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 54%|█████▍ | 5.40G/9.99G [00:28<00:20, 229MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 60%|██████ | 5.45G/9.06G [00:28<00:15, 226MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 54%|█████▍ | 5.43G/9.99G [00:28<00:19, 230MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 61%|██████ | 5.48G/9.06G [00:28<00:15, 228MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 55%|█████▍ | 5.46G/9.99G [00:28<00:19, 231MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 61%|██████ | 5.52G/9.06G [00:28<00:15, 228MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 55%|█████▌ | 5.49G/9.99G [00:28<00:19, 231MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 61%|██████ | 5.55G/9.06G [00:28<00:15, 229MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 55%|█████▌ | 5.53G/9.99G [00:28<00:19, 231MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 62%|██████▏ | 5.58G/9.06G [00:28<00:15, 229MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00001-of-00002.safetensors: 56%|█████▌ | 5.56G/9.99G [00:28<00:19, 230MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | +2024-01-10 14:43:26 | ERROR | stderr | model-00002-of-00002.safetensors: 62%|██████▏ | 5.61G/9.06G [00:28<00:15, 230MB/s] +2024-01-10 14:43:26 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 56%|█████▌ | 5.59G/9.99G [00:28<00:19, 230MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 62%|██████▏ | 5.64G/9.06G [00:29<00:14, 230MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 56%|█████▋ | 5.62G/9.99G [00:29<00:18, 231MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 63%|██████▎ | 5.67G/9.06G [00:29<00:14, 231MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 57%|█████▋ | 5.65G/9.99G [00:29<00:18, 232MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 63%|██████▎ | 5.70G/9.06G [00:29<00:14, 231MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 57%|█████▋ | 5.68G/9.99G [00:29<00:18, 232MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 63%|██████▎ | 5.74G/9.06G [00:29<00:14, 232MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 57%|█████▋ | 5.71G/9.99G [00:29<00:18, 231MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 64%|██████▎ | 5.77G/9.06G [00:29<00:14, 223MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 58%|█████▊ | 5.75G/9.99G [00:29<00:19, 222MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 64%|██████▍ | 5.80G/9.06G [00:29<00:14, 218MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00001-of-00002.safetensors: 58%|█████▊ | 5.78G/9.99G [00:29<00:19, 216MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | +2024-01-10 14:43:27 | ERROR | stderr | model-00002-of-00002.safetensors: 64%|██████▍ | 5.83G/9.06G [00:29<00:15, 214MB/s] +2024-01-10 14:43:27 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 58%|█████▊ | 5.81G/9.99G [00:29<00:19, 212MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 65%|██████▍ | 5.86G/9.06G [00:30<00:15, 211MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 58%|█████▊ | 5.84G/9.99G [00:30<00:19, 210MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 65%|██████▌ | 5.89G/9.06G [00:30<00:15, 208MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 59%|█████▉ | 5.87G/9.99G [00:30<00:19, 207MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 65%|██████▌ | 5.91G/9.06G [00:30<00:15, 206MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 59%|█████▉ | 5.90G/9.99G [00:30<00:19, 206MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 66%|██████▌ | 5.93G/9.06G [00:30<00:15, 199MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 59%|█████▉ | 5.93G/9.99G [00:30<00:19, 206MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 66%|██████▌ | 5.97G/9.06G [00:30<00:15, 200MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 60%|█████▉ | 5.97G/9.99G [00:30<00:19, 206MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 66%|██████▌ | 6.00G/9.06G [00:30<00:15, 201MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00001-of-00002.safetensors: 60%|██████ | 6.00G/9.99G [00:30<00:19, 204MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | +2024-01-10 14:43:28 | ERROR | stderr | model-00002-of-00002.safetensors: 67%|██████▋ | 6.03G/9.06G [00:30<00:14, 202MB/s] +2024-01-10 14:43:28 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 60%|██████ | 6.02G/9.99G [00:31<00:19, 199MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 67%|██████▋ | 6.06G/9.06G [00:31<00:14, 211MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 60%|██████ | 6.04G/9.99G [00:31<00:20, 197MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 67%|██████▋ | 6.09G/9.06G [00:31<00:14, 207MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 61%|██████ | 6.06G/9.99G [00:31<00:20, 193MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 67%|██████▋ | 6.11G/9.06G [00:31<00:14, 200MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 61%|██████ | 6.09G/9.99G [00:31<00:18, 206MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 61%|██████▏ | 6.13G/9.99G [00:31<00:15, 248MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 62%|██████▏ | 6.17G/9.99G [00:31<00:15, 249MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 68%|██████▊ | 6.13G/9.06G [00:31<00:21, 139MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 68%|██████▊ | 6.16G/9.06G [00:31<00:19, 148MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 62%|██████▏ | 6.20G/9.99G [00:31<00:16, 233MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00002-of-00002.safetensors: 68%|██████▊ | 6.19G/9.06G [00:31<00:16, 171MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:29 | ERROR | stderr | +2024-01-10 14:43:29 | ERROR | stderr | model-00001-of-00002.safetensors: 62%|██████▏ | 6.23G/9.99G [00:31<00:17, 215MB/s] +2024-01-10 14:43:29 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 69%|██████▊ | 6.22G/9.06G [00:32<00:15, 181MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 63%|██████▎ | 6.26G/9.99G [00:32<00:17, 212MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 69%|██████▉ | 6.25G/9.06G [00:32<00:14, 188MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 63%|██████▎ | 6.29G/9.99G [00:32<00:17, 210MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 69%|██████▉ | 6.28G/9.06G [00:32<00:14, 193MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 63%|██████▎ | 6.32G/9.99G [00:32<00:17, 208MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 70%|██████▉ | 6.31G/9.06G [00:32<00:13, 196MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 64%|██████▎ | 6.35G/9.99G [00:32<00:17, 207MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 70%|███████ | 6.34G/9.06G [00:32<00:13, 199MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 64%|██████▍ | 6.38G/9.99G [00:32<00:17, 201MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 70%|███████ | 6.38G/9.06G [00:32<00:13, 200MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00001-of-00002.safetensors: 64%|██████▍ | 6.41G/9.99G [00:32<00:17, 202MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | +2024-01-10 14:43:30 | ERROR | stderr | model-00002-of-00002.safetensors: 71%|███████ | 6.41G/9.06G [00:32<00:13, 202MB/s] +2024-01-10 14:43:30 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 64%|██████▍ | 6.44G/9.99G [00:33<00:17, 203MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 71%|███████ | 6.44G/9.06G [00:33<00:12, 205MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 65%|██████▍ | 6.47G/9.99G [00:33<00:16, 208MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 71%|███████▏ | 6.47G/9.06G [00:33<00:12, 211MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 65%|██████▌ | 6.50G/9.99G [00:33<00:16, 214MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 72%|███████▏ | 6.50G/9.06G [00:33<00:11, 217MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 65%|██████▌ | 6.53G/9.99G [00:33<00:15, 219MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 72%|███████▏ | 6.53G/9.06G [00:33<00:11, 221MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 66%|██████▌ | 6.56G/9.99G [00:33<00:15, 222MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 72%|███████▏ | 6.56G/9.06G [00:33<00:11, 222MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 66%|██████▌ | 6.60G/9.99G [00:33<00:15, 224MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 66%|██████▋ | 6.63G/9.99G [00:33<00:14, 236MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 73%|███████▎ | 6.60G/9.06G [00:33<00:11, 216MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00001-of-00002.safetensors: 67%|██████▋ | 6.66G/9.99G [00:33<00:14, 237MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | +2024-01-10 14:43:31 | ERROR | stderr | model-00002-of-00002.safetensors: 73%|███████▎ | 6.63G/9.06G [00:33<00:11, 220MB/s] +2024-01-10 14:43:31 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 67%|██████▋ | 6.69G/9.99G [00:34<00:13, 236MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 73%|███████▎ | 6.66G/9.06G [00:34<00:10, 224MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 67%|██████▋ | 6.72G/9.99G [00:34<00:13, 236MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 74%|███████▍ | 6.69G/9.06G [00:34<00:10, 226MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 68%|██████▊ | 6.75G/9.99G [00:34<00:13, 236MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 74%|███████▍ | 6.72G/9.06G [00:34<00:10, 230MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 68%|██████▊ | 6.78G/9.99G [00:34<00:13, 233MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 75%|███████▍ | 6.75G/9.06G [00:34<00:10, 231MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 75%|███████▍ | 6.78G/9.06G [00:34<00:09, 242MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 68%|██████▊ | 6.82G/9.99G [00:34<00:14, 223MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 75%|███████▌ | 6.82G/9.06G [00:34<00:09, 240MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 69%|██████▊ | 6.85G/9.99G [00:34<00:13, 225MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00002-of-00002.safetensors: 76%|███████▌ | 6.85G/9.06G [00:34<00:09, 237MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:32 | ERROR | stderr | +2024-01-10 14:43:32 | ERROR | stderr | model-00001-of-00002.safetensors: 69%|██████▉ | 6.88G/9.99G [00:34<00:13, 227MB/s] +2024-01-10 14:43:32 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 76%|███████▌ | 6.88G/9.06G [00:34<00:08, 247MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 76%|███████▋ | 6.92G/9.06G [00:35<00:07, 275MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 69%|██████▉ | 6.91G/9.99G [00:35<00:18, 164MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 77%|███████▋ | 6.95G/9.06G [00:35<00:07, 277MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 69%|██████▉ | 6.94G/9.99G [00:35<00:16, 185MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 77%|███████▋ | 6.98G/9.06G [00:35<00:07, 267MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 70%|██████▉ | 6.97G/9.99G [00:35<00:15, 198MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 78%|███████▊ | 7.03G/9.06G [00:35<00:07, 281MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 70%|███████ | 7.00G/9.99G [00:35<00:13, 217MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 78%|███████▊ | 7.06G/9.06G [00:35<00:07, 279MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 70%|███████ | 7.04G/9.99G [00:35<00:12, 236MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 78%|███████▊ | 7.09G/9.06G [00:35<00:07, 273MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 71%|███████ | 7.07G/9.99G [00:35<00:11, 245MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 79%|███████▊ | 7.12G/9.06G [00:35<00:06, 278MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00001-of-00002.safetensors: 71%|███████ | 7.10G/9.99G [00:35<00:11, 250MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | +2024-01-10 14:43:33 | ERROR | stderr | model-00002-of-00002.safetensors: 79%|███████▉ | 7.15G/9.06G [00:35<00:06, 280MB/s] +2024-01-10 14:43:33 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 71%|███████▏ | 7.13G/9.99G [00:36<00:11, 258MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 79%|███████▉ | 7.18G/9.06G [00:36<00:06, 279MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 72%|███████▏ | 7.16G/9.99G [00:36<00:10, 264MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 80%|███████▉ | 7.21G/9.06G [00:36<00:06, 280MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 72%|███████▏ | 7.19G/9.99G [00:36<00:10, 269MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 80%|███████▉ | 7.25G/9.06G [00:36<00:06, 279MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 72%|███████▏ | 7.22G/9.99G [00:36<00:10, 271MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 80%|████████ | 7.28G/9.06G [00:36<00:06, 279MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 73%|███████▎ | 7.26G/9.99G [00:36<00:10, 273MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 81%|████████ | 7.31G/9.06G [00:36<00:06, 279MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 73%|███████▎ | 7.29G/9.99G [00:36<00:09, 274MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 81%|████████ | 7.34G/9.06G [00:36<00:06, 278MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 73%|███████▎ | 7.32G/9.99G [00:36<00:09, 276MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 81%|████████▏ | 7.37G/9.06G [00:36<00:06, 278MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 74%|███████▎ | 7.35G/9.99G [00:36<00:09, 275MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 82%|████████▏ | 7.40G/9.06G [00:36<00:05, 279MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00001-of-00002.safetensors: 74%|███████▍ | 7.38G/9.99G [00:36<00:09, 277MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | +2024-01-10 14:43:34 | ERROR | stderr | model-00002-of-00002.safetensors: 82%|████████▏ | 7.43G/9.06G [00:36<00:05, 282MB/s] +2024-01-10 14:43:34 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 74%|███████▍ | 7.41G/9.99G [00:37<00:09, 281MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 82%|████████▏ | 7.47G/9.06G [00:37<00:05, 283MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 75%|███████▍ | 7.44G/9.99G [00:37<00:09, 283MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 83%|████████▎ | 7.50G/9.06G [00:37<00:05, 280MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 75%|███████▍ | 7.48G/9.99G [00:37<00:08, 280MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 83%|████████▎ | 7.53G/9.06G [00:37<00:05, 278MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 75%|███████▌ | 7.51G/9.99G [00:37<00:08, 279MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 83%|████████▎ | 7.56G/9.06G [00:37<00:05, 280MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 75%|███████▌ | 7.54G/9.99G [00:37<00:08, 279MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 84%|████████▍ | 7.59G/9.06G [00:37<00:05, 278MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 76%|███████▌ | 7.57G/9.99G [00:37<00:08, 269MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 84%|████████▍ | 7.62G/9.06G [00:37<00:05, 267MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 76%|███████▌ | 7.60G/9.99G [00:37<00:09, 260MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 84%|████████▍ | 7.65G/9.06G [00:37<00:05, 263MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00001-of-00002.safetensors: 76%|███████▋ | 7.63G/9.99G [00:37<00:08, 267MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | +2024-01-10 14:43:35 | ERROR | stderr | model-00002-of-00002.safetensors: 85%|████████▍ | 7.69G/9.06G [00:37<00:05, 270MB/s] +2024-01-10 14:43:35 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 77%|███████▋ | 7.67G/9.99G [00:37<00:08, 272MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 85%|████████▌ | 7.72G/9.06G [00:37<00:04, 274MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 77%|███████▋ | 7.70G/9.99G [00:38<00:08, 273MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 86%|████████▌ | 7.75G/9.06G [00:38<00:04, 276MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 77%|███████▋ | 7.73G/9.99G [00:38<00:08, 266MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 86%|████████▌ | 7.79G/9.06G [00:38<00:04, 291MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 78%|███████▊ | 7.76G/9.99G [00:38<00:08, 268MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 86%|████████▋ | 7.82G/9.06G [00:38<00:04, 288MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 78%|███████▊ | 7.79G/9.99G [00:38<00:08, 273MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 87%|████████▋ | 7.85G/9.06G [00:38<00:04, 288MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 78%|███████▊ | 7.82G/9.99G [00:38<00:07, 276MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 87%|████████▋ | 7.89G/9.06G [00:38<00:04, 289MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 79%|███████▊ | 7.85G/9.99G [00:38<00:07, 269MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 87%|████████▋ | 7.93G/9.06G [00:38<00:03, 298MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 79%|███████▉ | 7.90G/9.99G [00:38<00:07, 296MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 88%|████████▊ | 7.97G/9.06G [00:38<00:03, 313MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00001-of-00002.safetensors: 79%|███████▉ | 7.94G/9.99G [00:38<00:06, 311MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | +2024-01-10 14:43:36 | ERROR | stderr | model-00002-of-00002.safetensors: 88%|████████▊ | 8.01G/9.06G [00:38<00:03, 323MB/s] +2024-01-10 14:43:36 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 80%|███████▉ | 7.98G/9.99G [00:39<00:06, 322MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 89%|████████▉ | 8.05G/9.06G [00:39<00:03, 329MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 80%|████████ | 8.02G/9.99G [00:39<00:05, 336MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 89%|████████▉ | 8.10G/9.06G [00:39<00:02, 331MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 81%|████████ | 8.06G/9.99G [00:39<00:05, 338MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 90%|████████▉ | 8.14G/9.06G [00:39<00:02, 335MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 81%|████████ | 8.11G/9.99G [00:39<00:05, 341MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 90%|█████████ | 8.18G/9.06G [00:39<00:02, 344MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 82%|████████▏ | 8.15G/9.99G [00:39<00:05, 338MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 91%|█████████ | 8.22G/9.06G [00:39<00:02, 346MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 82%|████████▏ | 8.19G/9.99G [00:39<00:05, 349MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 91%|█████████ | 8.26G/9.06G [00:39<00:02, 349MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 82%|████████▏ | 8.23G/9.99G [00:39<00:05, 346MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 92%|█████████▏| 8.30G/9.06G [00:39<00:02, 352MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00001-of-00002.safetensors: 83%|████████▎ | 8.27G/9.99G [00:39<00:04, 348MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | +2024-01-10 14:43:37 | ERROR | stderr | model-00002-of-00002.safetensors: 92%|█████████▏| 8.35G/9.06G [00:39<00:02, 353MB/s] +2024-01-10 14:43:37 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 83%|████████▎ | 8.32G/9.99G [00:40<00:04, 345MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 93%|█████████▎| 8.39G/9.06G [00:40<00:01, 347MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 84%|████████▎ | 8.36G/9.99G [00:40<00:04, 351MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 93%|█████████▎| 8.43G/9.06G [00:40<00:01, 340MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 84%|████████▍ | 8.40G/9.99G [00:40<00:04, 351MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 94%|█████████▎| 8.47G/9.06G [00:40<00:01, 343MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 85%|████████▍ | 8.44G/9.99G [00:40<00:04, 354MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 94%|█████████▍| 8.51G/9.06G [00:40<00:01, 345MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 85%|████████▍ | 8.48G/9.99G [00:40<00:04, 357MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 94%|█████████▍| 8.56G/9.06G [00:40<00:01, 351MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 85%|████████▌ | 8.52G/9.99G [00:40<00:04, 356MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 95%|█████████▍| 8.60G/9.06G [00:40<00:01, 352MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 86%|████████▌ | 8.57G/9.99G [00:40<00:04, 349MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 95%|█████████▌| 8.64G/9.06G [00:40<00:01, 351MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 86%|████████▌ | 8.61G/9.99G [00:40<00:03, 348MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00002-of-00002.safetensors: 96%|█████████▌| 8.68G/9.06G [00:40<00:01, 349MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:38 | ERROR | stderr | +2024-01-10 14:43:38 | ERROR | stderr | model-00001-of-00002.safetensors: 87%|████████▋ | 8.65G/9.99G [00:40<00:03, 347MB/s] +2024-01-10 14:43:38 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 96%|█████████▋| 8.72G/9.06G [00:40<00:00, 348MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 87%|████████▋ | 8.69G/9.99G [00:41<00:03, 345MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 97%|█████████▋| 8.77G/9.06G [00:41<00:00, 351MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 97%|█████████▋| 8.81G/9.06G [00:41<00:00, 357MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 87%|████████▋ | 8.73G/9.99G [00:41<00:04, 290MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 98%|█████████▊| 8.85G/9.06G [00:41<00:00, 348MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 88%|████████▊ | 8.78G/9.99G [00:41<00:03, 307MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 88%|████████▊ | 8.82G/9.99G [00:41<00:03, 325MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 98%|█████████▊| 8.89G/9.06G [00:41<00:00, 292MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 89%|████████▊ | 8.86G/9.99G [00:41<00:03, 345MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 98%|█████████▊| 8.92G/9.06G [00:41<00:00, 276MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 89%|████████▉ | 8.90G/9.99G [00:41<00:03, 343MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 99%|█████████▉| 8.97G/9.06G [00:41<00:00, 296MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 90%|████████▉ | 8.94G/9.99G [00:41<00:03, 347MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00002-of-00002.safetensors: 99%|█████████▉| 9.01G/9.06G [00:41<00:00, 310MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:39 | ERROR | stderr | +2024-01-10 14:43:39 | ERROR | stderr | model-00001-of-00002.safetensors: 90%|████████▉ | 8.99G/9.99G [00:41<00:02, 347MB/s] +2024-01-10 14:43:39 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00002-of-00002.safetensors: 100%|█████████▉| 9.05G/9.06G [00:42<00:00, 320MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | model-00002-of-00002.safetensors: 100%|██████████| 9.06G/9.06G [00:42<00:00, 215MB/s] +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 90%|█████████ | 9.03G/9.99G [00:42<00:02, 346MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 91%|█████████ | 9.08G/9.99G [00:42<00:02, 374MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 91%|█████████▏| 9.13G/9.99G [00:42<00:02, 390MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 92%|█████████▏| 9.19G/9.99G [00:42<00:01, 406MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 92%|█████████▏| 9.24G/9.99G [00:42<00:01, 413MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 93%|█████████▎| 9.29G/9.99G [00:42<00:01, 417MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 94%|█████████▎| 9.34G/9.99G [00:42<00:01, 423MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:40 | ERROR | stderr | +2024-01-10 14:43:40 | ERROR | stderr | model-00001-of-00002.safetensors: 94%|█████████▍| 9.40G/9.99G [00:42<00:01, 420MB/s] +2024-01-10 14:43:40 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 95%|█████████▍| 9.45G/9.99G [00:43<00:01, 417MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 95%|█████████▌| 9.50G/9.99G [00:43<00:01, 421MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 96%|█████████▌| 9.55G/9.99G [00:43<00:01, 407MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 96%|█████████▌| 9.60G/9.99G [00:43<00:00, 410MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 97%|█████████▋| 9.66G/9.99G [00:43<00:00, 417MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 97%|█████████▋| 9.71G/9.99G [00:43<00:00, 421MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 98%|█████████▊| 9.76G/9.99G [00:43<00:00, 428MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:41 | ERROR | stderr | +2024-01-10 14:43:41 | ERROR | stderr | model-00001-of-00002.safetensors: 98%|█████████▊| 9.81G/9.99G [00:43<00:00, 427MB/s] +2024-01-10 14:43:41 | ERROR | stderr |  +2024-01-10 14:43:42 | ERROR | stderr | +2024-01-10 14:43:42 | ERROR | stderr | model-00001-of-00002.safetensors: 99%|█████████▉| 9.87G/9.99G [00:44<00:00, 432MB/s] +2024-01-10 14:43:42 | ERROR | stderr |  +2024-01-10 14:43:42 | ERROR | stderr | +2024-01-10 14:43:42 | ERROR | stderr | model-00001-of-00002.safetensors: 99%|█████████▉| 9.92G/9.99G [00:44<00:00, 428MB/s] +2024-01-10 14:43:42 | ERROR | stderr |  +2024-01-10 14:43:42 | ERROR | stderr | +2024-01-10 14:43:42 | ERROR | stderr | model-00001-of-00002.safetensors: 100%|█████████▉| 9.97G/9.99G [00:44<00:00, 427MB/s] +2024-01-10 14:43:42 | ERROR | stderr |  +2024-01-10 14:43:42 | ERROR | stderr | model-00001-of-00002.safetensors: 100%|██████████| 9.99G/9.99G [00:44<00:00, 225MB/s] +2024-01-10 14:43:42 | ERROR | stderr | +2024-01-10 14:43:42 | ERROR | stderr | Fetching 14 files: 29%|██▊ | 4/14 [00:44<02:10, 13.06s/it] +2024-01-10 14:43:42 | ERROR | stderr | Fetching 14 files: 100%|██████████| 14/14 [00:44<00:00, 3.18s/it] +2024-01-10 14:43:42 | ERROR | stderr | +2024-01-10 14:43:42 | ERROR | stderr | Loading pipeline components...: 0%| | 0/5 [00:00 https://141d1ff2611261e2ed.gradio.live +2024-01-10 14:51:08 | ERROR | stderr | +2024-01-10 14:51:08 | ERROR | stderr | model.safetensors: 91%|█████████ | 1.24G/1.36G [00:22<00:03, 38.3MB/s] +2024-01-10 14:51:08 | ERROR | stderr |  +2024-01-10 14:51:08 | ERROR | stderr | +2024-01-10 14:51:08 | ERROR | stderr | model.safetensors: 92%|█████████▏| 1.26G/1.36G [00:22<00:02, 40.1MB/s] +2024-01-10 14:51:08 | ERROR | stderr |  +2024-01-10 14:51:09 | ERROR | stderr | +2024-01-10 14:51:09 | ERROR | stderr | model.safetensors: 93%|█████████▎| 1.27G/1.36G [00:22<00:02, 40.6MB/s] +2024-01-10 14:51:09 | ERROR | stderr |  +2024-01-10 14:51:09 | ERROR | stderr | +2024-01-10 14:51:09 | ERROR | stderr | model.safetensors: 94%|█████████▍| 1.28G/1.36G [00:23<00:01, 46.3MB/s] +2024-01-10 14:51:09 | ERROR | stderr |  +2024-01-10 14:51:09 | ERROR | stderr | +2024-01-10 14:51:09 | ERROR | stderr | model.safetensors: 95%|█████████▍| 1.29G/1.36G [00:23<00:02, 35.1MB/s] +2024-01-10 14:51:09 | ERROR | stderr |  +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | model.safetensors: 96%|█████████▋| 1.31G/1.36G [00:24<00:01, 36.3MB/s] +2024-01-10 14:51:10 | ERROR | stderr |  +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | model.safetensors: 98%|█████████▊| 1.33G/1.36G [00:24<00:00, 52.0MB/s] +2024-01-10 14:51:10 | ERROR | stderr |  +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | model.safetensors: 99%|█████████▊| 1.34G/1.36G [00:24<00:00, 46.7MB/s] +2024-01-10 14:51:10 | ERROR | stderr |  +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | model.safetensors: 100%|██████████| 1.36G/1.36G [00:24<00:00, 61.5MB/s] +2024-01-10 14:51:10 | ERROR | stderr |  +2024-01-10 14:51:10 | ERROR | stderr | model.safetensors: 100%|██████████| 1.36G/1.36G [00:24<00:00, 55.0MB/s] +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | Fetching 13 files: 38%|███▊ | 5/13 [00:25<00:42, 5.31s/it] +2024-01-10 14:51:10 | ERROR | stderr | Fetching 13 files: 100%|██████████| 13/13 [00:25<00:00, 1.92s/it] +2024-01-10 14:51:10 | ERROR | stderr | +2024-01-10 14:51:10 | ERROR | stderr | Loading pipeline components...: 0%| | 0/6 [00:00 +2024-01-10 15:09:55 | ERROR | stderr | 0%| | 0/4 [00:00 +2024-01-10 15:11:14 | ERROR | stderr | 0%| | 0/20 [00:00 +2024-01-10 15:21:59 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 15:22:21 | ERROR | stderr | 0%| | 0/1 [00:00 +2024-01-10 15:35:35 | ERROR | stderr | 0%| | 0/1 [00:00 +2024-01-10 15:36:08 | ERROR | stderr | 0%| | 0/20 [00:00 +2024-01-10 15:41:32 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 15:41:50 | ERROR | stderr | 0%| | 0/1 [00:00 +2024-01-10 15:42:33 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 15:42:51 | ERROR | stderr | 0%| | 0/1 [00:00 +2024-01-10 15:47:36 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 15:47:53 | ERROR | stderr | 0%| | 0/1 [00:00 +2024-01-10 15:48:51 | ERROR | stderr | 0%| | 0/50 [00:00 https://ff60eb7a4f84d964ea.gradio.live +2024-01-10 15:59:19 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 15:59:19 | INFO | stdout | +2024-01-10 15:59:21 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 15:59:21 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 15:59:21 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 15:59:26 | INFO | stdout | INFO: 172.17.8.150:41986 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 15:59:26 | INFO | stdout | INFO: 172.17.8.150:45978 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 15:59:26 | INFO | stdout | INFO: 172.17.8.150:53652 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 15:59:26 | INFO | stdout | INFO: 172.17.8.150:60558 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 15:59:26 | INFO | stdout | INFO: 172.17.8.150:43188 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 15:59:27 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 15:59:33 | INFO | stdout | Running on public URL: https://8e939e888abf773be5.gradio.live +2024-01-10 15:59:33 | INFO | stdout | +2024-01-10 15:59:33 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 15:59:37 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 15:59:37 | ERROR | stderr | warnings.warn( +2024-01-10 15:59:37 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 15:59:37 | ERROR | stderr | warnings.warn( +2024-01-10 15:59:38 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 15:59:38 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 15:59:46 | INFO | stdout | INFO: 172.17.15.236:39962 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 15:59:46 | INFO | diffusion_infer | prompt: a dog +2024-01-10 15:59:46 | INFO | diffusion_infer | model.scheduler: DPMSolverMultistepScheduler { + "_class_name": "DPMSolverMultistepScheduler", + "_diffusers_version": "0.24.0", + "algorithm_type": "dpmsolver++", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "dynamic_thresholding_ratio": 0.995, + "euler_at_final": false, + "lambda_min_clipped": -Infinity, + "lower_order_final": true, + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": false, + "skip_prk_steps": true, + "solver_order": 2, + "solver_type": "midpoint", + "steps_offset": 1, + "thresholding": false, + "timestep_spacing": "linspace", + "trained_betas": null, + "use_karras_sigmas": false, + "use_lu_lambdas": false, + "variance_type": null +} + +2024-01-10 15:59:46 | INFO | diffusion_infer | model.type: +2024-01-10 15:59:49 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 16:00:17 | ERROR | stderr | 0%| | 0/20 [00:00 +2024-01-10 16:37:10 | ERROR | stderr | 0%| | 0/50 [00:00 +2024-01-10 16:37:28 | ERROR | stderr | 0%| | 0/4 [00:00 +2024-01-10 19:04:23 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:04:23 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:04:23 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:04:23 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 55, in +2024-01-10 19:04:23 | ERROR | stderr | from ...Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:04:23 | ERROR | stderr | ImportError: attempted relative import beyond top-level package +2024-01-10 19:06:55 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:06:55 | INFO | stdout | +2024-01-10 19:06:58 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:06:58 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:06:58 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:06:58 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:06:58 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:06:58 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:06:58 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:06:58 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:06:58 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:06:58 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 55, in +2024-01-10 19:06:58 | ERROR | stderr | from ..Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:06:58 | ERROR | stderr | ModuleNotFoundError: No module named 'fastchat.Lavie' +2024-01-10 19:17:11 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:17:11 | INFO | stdout | +2024-01-10 19:17:13 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:17:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:17:13 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:17:13 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:17:13 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:17:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:17:13 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:17:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:17:13 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:17:13 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 58, in +2024-01-10 19:17:13 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:17:13 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:18:17 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:18:17 | INFO | stdout | +2024-01-10 19:18:19 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:18:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:18:19 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:18:19 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:18:19 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:18:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:18:19 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:18:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:18:19 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:18:19 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 59, in +2024-01-10 19:18:19 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:18:19 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:19:14 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:19:14 | INFO | stdout | +2024-01-10 19:19:16 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:19:16 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:19:16 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:19:16 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:19:16 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:19:16 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:19:16 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:19:16 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:19:16 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:19:16 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:19:16 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 59, in +2024-01-10 19:19:16 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:19:16 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:19:44 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:19:44 | INFO | stdout | +2024-01-10 19:19:47 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:19:47 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:19:47 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:19:47 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:19:47 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:19:47 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:19:47 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:19:47 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:19:47 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:19:47 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:19:47 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 59, in +2024-01-10 19:19:47 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:19:47 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:21:36 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:21:36 | INFO | stdout | +2024-01-10 19:21:38 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:21:38 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:21:38 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:21:38 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:21:38 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:21:38 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:21:38 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:21:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:21:38 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:21:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:21:38 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:21:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:21:38 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:21:38 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:22:24 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:22:24 | INFO | stdout | +2024-01-10 19:22:26 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:22:26 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:22:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:22:26 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:22:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:22:26 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:22:26 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:22:26 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:22:26 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:22:26 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:22:26 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:22:26 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:22:26 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:23:26 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:23:26 | INFO | stdout | +2024-01-10 19:23:28 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:23:28 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:23:28 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:23:28 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:23:28 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:23:28 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:23:28 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:23:28 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:23:28 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:23:28 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:23:28 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:23:28 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:23:28 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:23:28 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:27:13 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:27:13 | INFO | stdout | +2024-01-10 19:27:15 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:27:15 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:27:15 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:27:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:27:15 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:27:15 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:27:15 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:27:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:27:15 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:27:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:27:15 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:27:15 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:27:15 | ERROR | stderr | from Lavie.base.models.unet import VideoGenPipeline +2024-01-10 19:27:15 | ERROR | stderr | ModuleNotFoundError: No module named 'Lavie' +2024-01-10 19:36:36 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:36:36 | INFO | stdout | +2024-01-10 19:36:38 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:36:38 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:36:38 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:36:38 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:36:38 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:36:38 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:36:38 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:36:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:36:38 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:36:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:36:38 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:36:38 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:36:38 | ERROR | stderr | from base.models.unet import VideoGenPipeline +2024-01-10 19:36:38 | ERROR | stderr | ModuleNotFoundError: No module named 'base' +2024-01-10 19:38:23 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:38:23 | INFO | stdout | +2024-01-10 19:38:25 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:38:25 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:38:25 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:38:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:38:25 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:38:25 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:38:25 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:38:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:38:25 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:38:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:38:25 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:38:25 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:38:25 | ERROR | stderr | from base.models.unet import VideoGenPipeline +2024-01-10 19:38:25 | ERROR | stderr | ModuleNotFoundError: No module named 'base' +2024-01-10 19:41:05 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:41:05 | INFO | stdout | +2024-01-10 19:41:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:41:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:41:07 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:41:07 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:41:07 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:41:07 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:41:07 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:41:07 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:41:07 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:41:07 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:41:07 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:41:07 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 60, in +2024-01-10 19:41:07 | ERROR | stderr | from unet import VideoGenPipeline +2024-01-10 19:41:07 | ERROR | stderr | ImportError: cannot import name 'VideoGenPipeline' from 'unet' (/h/tianleli/LaVie/base/models/unet.py) +2024-01-10 19:44:12 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:44:12 | INFO | stdout | +2024-01-10 19:44:14 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:44:14 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:44:14 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:44:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-10 19:44:14 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-10 19:44:14 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-10 19:44:14 | ERROR | stderr | exec(code, run_globals) +2024-01-10 19:44:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 20, in +2024-01-10 19:44:14 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:44:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/__init__.py", line 1, in +2024-01-10 19:44:14 | ERROR | stderr | from fastchat.model.model_adapter import ( +2024-01-10 19:44:14 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_adapter.py", line 62, in +2024-01-10 19:44:14 | ERROR | stderr | from .pipelines.pipeline_videogen import VideoGenPipeline +2024-01-10 19:44:14 | ERROR | stderr | ModuleNotFoundError: No module named 'fastchat.model.pipelines' +2024-01-10 19:45:05 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:45:05 | INFO | stdout | +2024-01-10 19:45:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:45:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:45:22 | INFO | stdout | Ema existing! +2024-01-10 19:45:26 | ERROR | stderr | INFO: Started server process [20402] +2024-01-10 19:45:26 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 19:45:26 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 19:45:26 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 19:47:15 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 19:47:15 | INFO | stdout | +2024-01-10 19:47:18 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 19:47:18 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 19:47:18 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 19:47:21 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:47:21 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:40240 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:45744 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:37818 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:49834 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:45758 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:22 | INFO | stdout | INFO: 172.17.8.150:48542 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 19:47:23 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 19:47:36 | INFO | stdout | Running on public URL: https://5f40c916798cc78429.gradio.live +2024-01-10 19:47:36 | INFO | stdout | +2024-01-10 19:47:36 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 19:47:40 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 19:47:40 | ERROR | stderr | warnings.warn( +2024-01-10 19:47:40 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 19:47:40 | ERROR | stderr | warnings.warn( +2024-01-10 19:47:45 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 19:47:45 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 19:47:55 | INFO | stdout | INFO: 172.17.15.236:46062 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 19:47:55 | ERROR | stderr | ERROR: Exception in ASGI application +2024-01-10 19:47:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 19:47:55 | ERROR | stderr | await func() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 232, in listen_for_disconnect +2024-01-10 19:47:55 | ERROR | stderr | message = await receive() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 538, in receive +2024-01-10 19:47:55 | ERROR | stderr | await self.message_event.wait() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/locks.py", line 226, in wait +2024-01-10 19:47:55 | ERROR | stderr | await fut +2024-01-10 19:47:55 | ERROR | stderr | asyncio.exceptions.CancelledError: Cancelled by cancel scope 7ff5bca380a0 +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | + Exception Group Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi +2024-01-10 19:47:55 | ERROR | stderr | | result = await app( # type: ignore[func-returns-value] +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | return await self.app(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/fastapi/applications.py", line 1054, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await super().__call__(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/applications.py", line 116, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await self.middleware_stack(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 186, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | raise exc +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 164, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await self.app(scope, receive, _send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 62, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 19:47:55 | ERROR | stderr | | raise exc +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 19:47:55 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 746, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await route.handle(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 288, in handle +2024-01-10 19:47:55 | ERROR | stderr | | await self.app(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 75, in app +2024-01-10 19:47:55 | ERROR | stderr | | await wrap_app_handling_exceptions(app, request)(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 19:47:55 | ERROR | stderr | | raise exc +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 19:47:55 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 73, in app +2024-01-10 19:47:55 | ERROR | stderr | | await response(scope, receive, send) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 19:47:55 | ERROR | stderr | | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 678, in __aexit__ +2024-01-10 19:47:55 | ERROR | stderr | | raise BaseExceptionGroup( +2024-01-10 19:47:55 | ERROR | stderr | | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception) +2024-01-10 19:47:55 | ERROR | stderr | +-+---------------- 1 ---------------- +2024-01-10 19:47:55 | ERROR | stderr | | Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 19:47:55 | ERROR | stderr | | await func() +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 244, in stream_response +2024-01-10 19:47:55 | ERROR | stderr | | async for chunk in self.body_iterator: +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 57, in iterate_in_threadpool +2024-01-10 19:47:55 | ERROR | stderr | | yield await anyio.to_thread.run_sync(_next, iterator) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 19:47:55 | ERROR | stderr | | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 19:47:55 | ERROR | stderr | | return await future +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 19:47:55 | ERROR | stderr | | result = context.run(func, *args) +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 47, in _next +2024-01-10 19:47:55 | ERROR | stderr | | return next(iterator) +2024-01-10 19:47:55 | ERROR | stderr | | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 122, in generate_stream_gate +2024-01-10 19:47:55 | ERROR | stderr | | for output in self.generate_stream_func( +2024-01-10 19:47:55 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 35, in generator_context +2024-01-10 19:47:55 | ERROR | stderr | | response = gen.send(None) +2024-01-10 19:47:55 | ERROR | stderr | | File "/fs01/home/tianleli/VisualArena/fastchat/serve/inference.py", line 94, in generate_stream +2024-01-10 19:47:55 | ERROR | stderr | | if model.config.is_encoder_decoder: +2024-01-10 19:47:55 | ERROR | stderr | | AttributeError: 'FrozenDict' object has no attribute 'is_encoder_decoder' +2024-01-10 19:47:55 | ERROR | stderr | +------------------------------------ +2024-01-10 19:47:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 712, in _error_catcher +2024-01-10 19:47:55 | ERROR | stderr | yield +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1071, in read_chunked +2024-01-10 19:47:55 | ERROR | stderr | self._update_chunk_length() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1006, in _update_chunk_length +2024-01-10 19:47:55 | ERROR | stderr | raise InvalidChunkLength(self, line) from None +2024-01-10 19:47:55 | ERROR | stderr | urllib3.exceptions.InvalidChunkLength: InvalidChunkLength(got length b'', 0 bytes read) +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 816, in generate +2024-01-10 19:47:55 | ERROR | stderr | yield from self.raw.stream(chunk_size, decode_content=True) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 931, in stream +2024-01-10 19:47:55 | ERROR | stderr | yield from self.read_chunked(amt, decode_content=decode_content) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1100, in read_chunked +2024-01-10 19:47:55 | ERROR | stderr | self._original_response.close() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/contextlib.py", line 137, in __exit__ +2024-01-10 19:47:55 | ERROR | stderr | self.gen.throw(typ, value, traceback) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 729, in _error_catcher +2024-01-10 19:47:55 | ERROR | stderr | raise ProtocolError(f"Connection broken: {e!r}", e) from e +2024-01-10 19:47:55 | ERROR | stderr | urllib3.exceptions.ProtocolError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 19:47:55 | ERROR | stderr | +2024-01-10 19:47:55 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/queueing.py", line 407, in call_prediction +2024-01-10 19:47:55 | ERROR | stderr | output = await route_utils.call_process_api( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/route_utils.py", line 226, in call_process_api +2024-01-10 19:47:55 | ERROR | stderr | output = await app.get_blocks().process_api( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1550, in process_api +2024-01-10 19:47:55 | ERROR | stderr | result = await self.call_function( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1199, in call_function +2024-01-10 19:47:55 | ERROR | stderr | prediction = await utils.async_iteration(iterator) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 519, in async_iteration +2024-01-10 19:47:55 | ERROR | stderr | return await iterator.__anext__() +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 512, in __anext__ +2024-01-10 19:47:55 | ERROR | stderr | return await anyio.to_thread.run_sync( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 19:47:55 | ERROR | stderr | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 19:47:55 | ERROR | stderr | return await future +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 19:47:55 | ERROR | stderr | result = context.run(func, *args) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 495, in run_sync_iterator_async +2024-01-10 19:47:55 | ERROR | stderr | return next(iterator) +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 649, in gen_wrapper +2024-01-10 19:47:55 | ERROR | stderr | yield from f(*args, **kwargs) +2024-01-10 19:47:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 506, in diffusion_response +2024-01-10 19:47:55 | ERROR | stderr | for i, data in enumerate(stream_iter): +2024-01-10 19:47:55 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 399, in model_diffusion_worker_stream_iter +2024-01-10 19:47:55 | ERROR | stderr | for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 865, in iter_lines +2024-01-10 19:47:55 | ERROR | stderr | for chunk in self.iter_content( +2024-01-10 19:47:55 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 818, in generate +2024-01-10 19:47:55 | ERROR | stderr | raise ChunkedEncodingError(e) +2024-01-10 19:47:55 | ERROR | stderr | requests.exceptions.ChunkedEncodingError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 20:06:56 | ERROR | stderr | INFO: Shutting down +2024-01-10 20:06:56 | ERROR | stderr | INFO: Waiting for application shutdown. +2024-01-10 20:06:56 | ERROR | stderr | INFO: Application shutdown complete. +2024-01-10 20:06:56 | ERROR | stderr | INFO: Finished server process [20402] +2024-01-10 20:07:05 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:07:05 | INFO | stdout | +2024-01-10 20:07:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:07:07 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:07:21 | INFO | stdout | Ema existing! +2024-01-10 20:07:25 | ERROR | stderr | INFO: Started server process [11856] +2024-01-10 20:07:25 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:07:25 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:07:25 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:07:30 | INFO | stdout | Keyboard interruption in main thread... closing server. +2024-01-10 20:07:31 | INFO | stdout | Killing tunnel localhost:7861 <> https://5f40c916798cc78429.gradio.live +2024-01-10 20:07:57 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:07:57 | INFO | stdout | +2024-01-10 20:08:00 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:08:00 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:08:00 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:08:03 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:08:03 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:51610 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:59838 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:58978 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:39546 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:43882 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | INFO: 172.17.8.150:49004 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:08:04 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:08:12 | INFO | stdout | Running on public URL: https://42126c86d48fb75199.gradio.live +2024-01-10 20:08:12 | INFO | stdout | +2024-01-10 20:08:12 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:08:17 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:08:17 | ERROR | stderr | warnings.warn( +2024-01-10 20:08:17 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:08:17 | ERROR | stderr | warnings.warn( +2024-01-10 20:08:17 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:08:17 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:08:26 | INFO | stdout | INFO: 172.17.15.236:55976 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:08:26 | ERROR | stderr | ERROR: Exception in ASGI application +2024-01-10 20:08:26 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 20:08:26 | ERROR | stderr | await func() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 232, in listen_for_disconnect +2024-01-10 20:08:26 | ERROR | stderr | message = await receive() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 538, in receive +2024-01-10 20:08:26 | ERROR | stderr | await self.message_event.wait() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/locks.py", line 226, in wait +2024-01-10 20:08:26 | ERROR | stderr | await fut +2024-01-10 20:08:26 | ERROR | stderr | asyncio.exceptions.CancelledError: Cancelled by cancel scope 7f034f059460 +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | + Exception Group Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi +2024-01-10 20:08:26 | ERROR | stderr | | result = await app( # type: ignore[func-returns-value] +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | return await self.app(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/fastapi/applications.py", line 1054, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await super().__call__(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/applications.py", line 116, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await self.middleware_stack(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 186, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | raise exc +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 164, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await self.app(scope, receive, _send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 62, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 20:08:26 | ERROR | stderr | | raise exc +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 20:08:26 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 746, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await route.handle(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 288, in handle +2024-01-10 20:08:26 | ERROR | stderr | | await self.app(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 75, in app +2024-01-10 20:08:26 | ERROR | stderr | | await wrap_app_handling_exceptions(app, request)(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 20:08:26 | ERROR | stderr | | raise exc +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 20:08:26 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 73, in app +2024-01-10 20:08:26 | ERROR | stderr | | await response(scope, receive, send) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 20:08:26 | ERROR | stderr | | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 678, in __aexit__ +2024-01-10 20:08:26 | ERROR | stderr | | raise BaseExceptionGroup( +2024-01-10 20:08:26 | ERROR | stderr | | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception) +2024-01-10 20:08:26 | ERROR | stderr | +-+---------------- 1 ---------------- +2024-01-10 20:08:26 | ERROR | stderr | | Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 20:08:26 | ERROR | stderr | | await func() +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 244, in stream_response +2024-01-10 20:08:26 | ERROR | stderr | | async for chunk in self.body_iterator: +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 57, in iterate_in_threadpool +2024-01-10 20:08:26 | ERROR | stderr | | yield await anyio.to_thread.run_sync(_next, iterator) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 20:08:26 | ERROR | stderr | | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 20:08:26 | ERROR | stderr | | return await future +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 20:08:26 | ERROR | stderr | | result = context.run(func, *args) +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 47, in _next +2024-01-10 20:08:26 | ERROR | stderr | | return next(iterator) +2024-01-10 20:08:26 | ERROR | stderr | | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 122, in generate_stream_gate +2024-01-10 20:08:26 | ERROR | stderr | | for output in self.generate_stream_func( +2024-01-10 20:08:26 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 26, in generator_context +2024-01-10 20:08:26 | ERROR | stderr | | gen = func(*args, **kwargs) +2024-01-10 20:08:26 | ERROR | stderr | | TypeError: generate_stream_lavie() takes 4 positional arguments but 6 were given +2024-01-10 20:08:26 | ERROR | stderr | +------------------------------------ +2024-01-10 20:08:26 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 712, in _error_catcher +2024-01-10 20:08:26 | ERROR | stderr | yield +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1071, in read_chunked +2024-01-10 20:08:26 | ERROR | stderr | self._update_chunk_length() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1006, in _update_chunk_length +2024-01-10 20:08:26 | ERROR | stderr | raise InvalidChunkLength(self, line) from None +2024-01-10 20:08:26 | ERROR | stderr | urllib3.exceptions.InvalidChunkLength: InvalidChunkLength(got length b'', 0 bytes read) +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 816, in generate +2024-01-10 20:08:26 | ERROR | stderr | yield from self.raw.stream(chunk_size, decode_content=True) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 931, in stream +2024-01-10 20:08:26 | ERROR | stderr | yield from self.read_chunked(amt, decode_content=decode_content) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1100, in read_chunked +2024-01-10 20:08:26 | ERROR | stderr | self._original_response.close() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/contextlib.py", line 137, in __exit__ +2024-01-10 20:08:26 | ERROR | stderr | self.gen.throw(typ, value, traceback) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 729, in _error_catcher +2024-01-10 20:08:26 | ERROR | stderr | raise ProtocolError(f"Connection broken: {e!r}", e) from e +2024-01-10 20:08:26 | ERROR | stderr | urllib3.exceptions.ProtocolError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 20:08:26 | ERROR | stderr | +2024-01-10 20:08:26 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/queueing.py", line 407, in call_prediction +2024-01-10 20:08:26 | ERROR | stderr | output = await route_utils.call_process_api( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/route_utils.py", line 226, in call_process_api +2024-01-10 20:08:26 | ERROR | stderr | output = await app.get_blocks().process_api( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1550, in process_api +2024-01-10 20:08:26 | ERROR | stderr | result = await self.call_function( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1199, in call_function +2024-01-10 20:08:26 | ERROR | stderr | prediction = await utils.async_iteration(iterator) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 519, in async_iteration +2024-01-10 20:08:26 | ERROR | stderr | return await iterator.__anext__() +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 512, in __anext__ +2024-01-10 20:08:26 | ERROR | stderr | return await anyio.to_thread.run_sync( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 20:08:26 | ERROR | stderr | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 20:08:26 | ERROR | stderr | return await future +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 20:08:26 | ERROR | stderr | result = context.run(func, *args) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 495, in run_sync_iterator_async +2024-01-10 20:08:26 | ERROR | stderr | return next(iterator) +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 649, in gen_wrapper +2024-01-10 20:08:26 | ERROR | stderr | yield from f(*args, **kwargs) +2024-01-10 20:08:26 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 506, in diffusion_response +2024-01-10 20:08:26 | ERROR | stderr | for i, data in enumerate(stream_iter): +2024-01-10 20:08:26 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 399, in model_diffusion_worker_stream_iter +2024-01-10 20:08:26 | ERROR | stderr | for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 865, in iter_lines +2024-01-10 20:08:26 | ERROR | stderr | for chunk in self.iter_content( +2024-01-10 20:08:26 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 818, in generate +2024-01-10 20:08:26 | ERROR | stderr | raise ChunkedEncodingError(e) +2024-01-10 20:08:26 | ERROR | stderr | requests.exceptions.ChunkedEncodingError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 20:09:43 | ERROR | stderr | INFO: Shutting down +2024-01-10 20:09:43 | ERROR | stderr | INFO: Waiting for application shutdown. +2024-01-10 20:09:43 | ERROR | stderr | INFO: Application shutdown complete. +2024-01-10 20:09:43 | ERROR | stderr | INFO: Finished server process [11856] +2024-01-10 20:09:54 | INFO | stdout | Keyboard interruption in main thread... closing server. +2024-01-10 20:09:54 | INFO | stdout | Killing tunnel localhost:7861 <> https://42126c86d48fb75199.gradio.live +2024-01-10 20:09:57 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:09:57 | INFO | stdout | +2024-01-10 20:09:59 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:09:59 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:10:13 | INFO | stdout | Ema existing! +2024-01-10 20:10:17 | ERROR | stderr | INFO: Started server process [14380] +2024-01-10 20:10:17 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:10:17 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:10:17 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:10:44 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:10:44 | INFO | stdout | +2024-01-10 20:10:46 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:10:46 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:10:46 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:10:50 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:10:50 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:38570 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:38294 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:47540 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:46324 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:56024 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | INFO: 172.17.8.150:40878 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:10:51 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:10:56 | INFO | stdout | Running on public URL: https://ce3c155c342a2f19d2.gradio.live +2024-01-10 20:10:56 | INFO | stdout | +2024-01-10 20:10:56 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:11:18 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:11:18 | ERROR | stderr | warnings.warn( +2024-01-10 20:11:18 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:11:18 | ERROR | stderr | warnings.warn( +2024-01-10 20:11:24 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:11:24 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:11:24 | INFO | stdout | INFO: 172.17.15.236:40808 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:11:24 | INFO | diffusion_infer | prompt: a dog +2024-01-10 20:11:24 | INFO | diffusion_infer | model.scheduler: LCMScheduler { + "_class_name": "LCMScheduler", + "_diffusers_version": "0.24.0", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 1000, + "original_inference_steps": 50, + "prediction_type": "epsilon", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": false, + "timestep_scaling": 10.0, + "timestep_spacing": "leading", + "trained_betas": null +} + +2024-01-10 20:11:24 | INFO | diffusion_infer | model.type: +2024-01-10 20:11:24 | ERROR | stderr | 0%| | 0/4 [00:00 https://ce3c155c342a2f19d2.gradio.live +2024-01-10 20:13:16 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:13:16 | INFO | stdout | +2024-01-10 20:13:18 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:13:18 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:13:18 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:13:22 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:13:22 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:44170 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:55520 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:33776 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:45414 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:42632 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | INFO: 172.17.8.150:45578 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:13:23 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:13:31 | INFO | stdout | Running on public URL: https://f9ab9281d350eefae9.gradio.live +2024-01-10 20:13:31 | INFO | stdout | +2024-01-10 20:13:31 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:13:35 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:13:35 | ERROR | stderr | warnings.warn( +2024-01-10 20:13:35 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:13:35 | ERROR | stderr | warnings.warn( +2024-01-10 20:13:35 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:13:35 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:13:46 | INFO | stdout | INFO: 172.17.15.236:58052 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:13:46 | INFO | diffusion_infer | prompt: a running dog +2024-01-10 20:13:46 | ERROR | stderr | ERROR: Exception in ASGI application +2024-01-10 20:13:46 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 20:13:46 | ERROR | stderr | await func() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 232, in listen_for_disconnect +2024-01-10 20:13:46 | ERROR | stderr | message = await receive() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 538, in receive +2024-01-10 20:13:46 | ERROR | stderr | await self.message_event.wait() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/locks.py", line 226, in wait +2024-01-10 20:13:46 | ERROR | stderr | await fut +2024-01-10 20:13:46 | ERROR | stderr | asyncio.exceptions.CancelledError: Cancelled by cancel scope 7f6ad0225b20 +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | + Exception Group Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi +2024-01-10 20:13:46 | ERROR | stderr | | result = await app( # type: ignore[func-returns-value] +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | return await self.app(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/fastapi/applications.py", line 1054, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await super().__call__(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/applications.py", line 116, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await self.middleware_stack(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 186, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | raise exc +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/errors.py", line 164, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await self.app(scope, receive, _send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 62, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 20:13:46 | ERROR | stderr | | raise exc +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 20:13:46 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 746, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await route.handle(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 288, in handle +2024-01-10 20:13:46 | ERROR | stderr | | await self.app(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 75, in app +2024-01-10 20:13:46 | ERROR | stderr | | await wrap_app_handling_exceptions(app, request)(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 55, in wrapped_app +2024-01-10 20:13:46 | ERROR | stderr | | raise exc +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/_exception_handler.py", line 44, in wrapped_app +2024-01-10 20:13:46 | ERROR | stderr | | await app(scope, receive, sender) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/routing.py", line 73, in app +2024-01-10 20:13:46 | ERROR | stderr | | await response(scope, receive, send) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 259, in __call__ +2024-01-10 20:13:46 | ERROR | stderr | | await wrap(partial(self.listen_for_disconnect, receive)) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 678, in __aexit__ +2024-01-10 20:13:46 | ERROR | stderr | | raise BaseExceptionGroup( +2024-01-10 20:13:46 | ERROR | stderr | | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception) +2024-01-10 20:13:46 | ERROR | stderr | +-+---------------- 1 ---------------- +2024-01-10 20:13:46 | ERROR | stderr | | Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 255, in wrap +2024-01-10 20:13:46 | ERROR | stderr | | await func() +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/responses.py", line 244, in stream_response +2024-01-10 20:13:46 | ERROR | stderr | | async for chunk in self.body_iterator: +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 57, in iterate_in_threadpool +2024-01-10 20:13:46 | ERROR | stderr | | yield await anyio.to_thread.run_sync(_next, iterator) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 20:13:46 | ERROR | stderr | | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 20:13:46 | ERROR | stderr | | return await future +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 20:13:46 | ERROR | stderr | | result = context.run(func, *args) +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/starlette/concurrency.py", line 47, in _next +2024-01-10 20:13:46 | ERROR | stderr | | return next(iterator) +2024-01-10 20:13:46 | ERROR | stderr | | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 122, in generate_stream_gate +2024-01-10 20:13:46 | ERROR | stderr | | for output in self.generate_stream_func( +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 35, in generator_context +2024-01-10 20:13:46 | ERROR | stderr | | response = gen.send(None) +2024-01-10 20:13:46 | ERROR | stderr | | File "/fs01/home/tianleli/VisualArena/fastchat/model/model_lavie.py", line 25, in generate_stream_lavie +2024-01-10 20:13:46 | ERROR | stderr | | logger.info(f"model.scheduler: {model.pipe.scheduler}") +2024-01-10 20:13:46 | ERROR | stderr | | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/diffusers/configuration_utils.py", line 138, in __getattr__ +2024-01-10 20:13:46 | ERROR | stderr | | raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") +2024-01-10 20:13:46 | ERROR | stderr | | AttributeError: 'VideoGenPipeline' object has no attribute 'pipe' +2024-01-10 20:13:46 | ERROR | stderr | +------------------------------------ +2024-01-10 20:13:46 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 712, in _error_catcher +2024-01-10 20:13:46 | ERROR | stderr | yield +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1071, in read_chunked +2024-01-10 20:13:46 | ERROR | stderr | self._update_chunk_length() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1006, in _update_chunk_length +2024-01-10 20:13:46 | ERROR | stderr | raise InvalidChunkLength(self, line) from None +2024-01-10 20:13:46 | ERROR | stderr | urllib3.exceptions.InvalidChunkLength: InvalidChunkLength(got length b'', 0 bytes read) +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 816, in generate +2024-01-10 20:13:46 | ERROR | stderr | yield from self.raw.stream(chunk_size, decode_content=True) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 931, in stream +2024-01-10 20:13:46 | ERROR | stderr | yield from self.read_chunked(amt, decode_content=decode_content) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 1100, in read_chunked +2024-01-10 20:13:46 | ERROR | stderr | self._original_response.close() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/contextlib.py", line 137, in __exit__ +2024-01-10 20:13:46 | ERROR | stderr | self.gen.throw(typ, value, traceback) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/response.py", line 729, in _error_catcher +2024-01-10 20:13:46 | ERROR | stderr | raise ProtocolError(f"Connection broken: {e!r}", e) from e +2024-01-10 20:13:46 | ERROR | stderr | urllib3.exceptions.ProtocolError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-10 20:13:46 | ERROR | stderr | +2024-01-10 20:13:46 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/queueing.py", line 407, in call_prediction +2024-01-10 20:13:46 | ERROR | stderr | output = await route_utils.call_process_api( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/route_utils.py", line 226, in call_process_api +2024-01-10 20:13:46 | ERROR | stderr | output = await app.get_blocks().process_api( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1550, in process_api +2024-01-10 20:13:46 | ERROR | stderr | result = await self.call_function( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/blocks.py", line 1199, in call_function +2024-01-10 20:13:46 | ERROR | stderr | prediction = await utils.async_iteration(iterator) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 519, in async_iteration +2024-01-10 20:13:46 | ERROR | stderr | return await iterator.__anext__() +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 512, in __anext__ +2024-01-10 20:13:46 | ERROR | stderr | return await anyio.to_thread.run_sync( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync +2024-01-10 20:13:46 | ERROR | stderr | return await get_async_backend().run_sync_in_worker_thread( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2134, in run_sync_in_worker_thread +2024-01-10 20:13:46 | ERROR | stderr | return await future +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 851, in run +2024-01-10 20:13:46 | ERROR | stderr | result = context.run(func, *args) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 495, in run_sync_iterator_async +2024-01-10 20:13:46 | ERROR | stderr | return next(iterator) +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/utils.py", line 649, in gen_wrapper +2024-01-10 20:13:46 | ERROR | stderr | yield from f(*args, **kwargs) +2024-01-10 20:13:46 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 506, in diffusion_response +2024-01-10 20:13:46 | ERROR | stderr | for i, data in enumerate(stream_iter): +2024-01-10 20:13:46 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/gradio_web_server.py", line 399, in model_diffusion_worker_stream_iter +2024-01-10 20:13:46 | ERROR | stderr | for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 865, in iter_lines +2024-01-10 20:13:46 | ERROR | stderr | for chunk in self.iter_content( +2024-01-10 20:13:46 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/models.py", line 818, in generate +2024-01-10 20:13:46 | ERROR | stderr | raise ChunkedEncodingError(e) +2024-01-10 20:13:46 | ERROR | stderr | requests.exceptions.ChunkedEncodingError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read)) +2024-01-10 20:14:27 | ERROR | stderr | INFO: Shutting down +2024-01-10 20:14:27 | ERROR | stderr | INFO: Waiting for application shutdown. +2024-01-10 20:14:27 | ERROR | stderr | INFO: Application shutdown complete. +2024-01-10 20:14:27 | ERROR | stderr | INFO: Finished server process [14380] +2024-01-10 20:14:34 | INFO | stdout | Keyboard interruption in main thread... closing server. +2024-01-10 20:14:34 | INFO | stdout | Killing tunnel localhost:7861 <> https://f9ab9281d350eefae9.gradio.live +2024-01-10 20:14:35 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:14:35 | INFO | stdout | +2024-01-10 20:14:37 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:14:37 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:14:51 | INFO | stdout | Ema existing! +2024-01-10 20:14:55 | ERROR | stderr | INFO: Started server process [20119] +2024-01-10 20:14:55 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:14:55 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:14:55 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:15:22 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:15:22 | INFO | stdout | +2024-01-10 20:15:25 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:15:25 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:15:25 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:15:28 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:15:28 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:41392 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:57946 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:36174 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:51466 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:33324 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | INFO: 172.17.8.150:52486 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:15:29 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:15:30 | INFO | stdout | Running on public URL: https://3837ca7162fbab7fc2.gradio.live +2024-01-10 20:15:30 | INFO | stdout | +2024-01-10 20:15:30 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:15:34 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:15:34 | ERROR | stderr | warnings.warn( +2024-01-10 20:15:34 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:15:34 | ERROR | stderr | warnings.warn( +2024-01-10 20:15:35 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:15:35 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:15:51 | INFO | stdout | INFO: 172.17.15.236:58462 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:15:51 | INFO | diffusion_infer | prompt: a running dog +2024-01-10 20:15:52 | ERROR | stderr | 0%| | 0/50 [00:00 https://3837ca7162fbab7fc2.gradio.live +2024-01-10 20:26:04 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:26:04 | INFO | stdout | +2024-01-10 20:26:06 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:26:06 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:26:20 | INFO | stdout | Ema existing! +2024-01-10 20:26:24 | ERROR | stderr | INFO: Started server process [31820] +2024-01-10 20:26:24 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:26:24 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:26:24 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:26:46 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:26:46 | INFO | stdout | +2024-01-10 20:26:48 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:26:48 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:26:48 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:26:51 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:26:51 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:50346 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:39928 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:34324 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:34078 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:49946 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:51 | INFO | stdout | INFO: 172.17.8.150:38714 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:26:52 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:26:53 | INFO | stdout | Running on public URL: https://f12b165be29d52e3e9.gradio.live +2024-01-10 20:26:53 | INFO | stdout | +2024-01-10 20:26:53 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:26:57 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:26:57 | ERROR | stderr | warnings.warn( +2024-01-10 20:26:57 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:26:57 | ERROR | stderr | warnings.warn( +2024-01-10 20:27:03 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:27:03 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:27:10 | INFO | stdout | INFO: 172.17.15.236:45130 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:27:10 | INFO | diffusion_infer | prompt: a running dog +2024-01-10 20:27:11 | ERROR | stderr | 0%| | 0/50 [00:00 https://f12b165be29d52e3e9.gradio.live +2024-01-10 20:31:26 | ERROR | stderr | INFO: Shutting down +2024-01-10 20:31:26 | ERROR | stderr | INFO: Waiting for application shutdown. +2024-01-10 20:31:26 | ERROR | stderr | INFO: Application shutdown complete. +2024-01-10 20:31:26 | ERROR | stderr | INFO: Finished server process [31820] +2024-01-10 20:31:33 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:31:33 | INFO | stdout | +2024-01-10 20:31:35 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:31:35 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:31:49 | INFO | stdout | Ema existing! +2024-01-10 20:31:53 | ERROR | stderr | INFO: Started server process [6268] +2024-01-10 20:31:53 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:31:53 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:31:53 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:31:59 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:31:59 | INFO | stdout | +2024-01-10 20:32:01 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:32:01 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:32:01 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:32:04 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:32:04 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:36288 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:41990 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:52202 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:60358 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:47516 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | INFO: 172.17.8.150:59132 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:32:04 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:32:09 | INFO | stdout | Running on public URL: https://fd2b61a15f0d7d8b17.gradio.live +2024-01-10 20:32:09 | INFO | stdout | +2024-01-10 20:32:09 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:32:14 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:32:14 | ERROR | stderr | warnings.warn( +2024-01-10 20:32:14 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:32:14 | ERROR | stderr | warnings.warn( +2024-01-10 20:32:27 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:32:27 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:32:33 | INFO | stdout | INFO: 172.17.15.236:48884 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:32:33 | INFO | diffusion_infer | prompt: a dog +2024-01-10 20:32:34 | ERROR | stderr | 0%| | 0/50 [00:00 https://fd2b61a15f0d7d8b17.gradio.live +2024-01-10 20:39:23 | ERROR | stderr | Error in atexit._run_exitfuncs: +2024-01-10 20:39:23 | ERROR | stderr | Traceback (most recent call last): +2024-01-10 20:39:23 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/torch/utils/data/_utils/__init__.py", line 45, in _set_python_exit_flag +2024-01-10 20:39:23 | ERROR | stderr | +2024-01-10 20:39:26 | INFO | stdout | Ema existing! +2024-01-10 20:39:30 | ERROR | stderr | INFO: Started server process [13373] +2024-01-10 20:39:30 | ERROR | stderr | INFO: Waiting for application startup. +2024-01-10 20:39:30 | ERROR | stderr | INFO: Application startup complete. +2024-01-10 20:39:30 | ERROR | stderr | INFO: Uvicorn running on http://172.17.8.39:31009 (Press CTRL+C to quit) +2024-01-10 20:39:46 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-10 20:39:46 | INFO | stdout | +2024-01-10 20:39:47 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable. +2024-01-10 20:39:47 | ERROR | stderr | warn("The installed version of bitsandbytes was compiled without GPU support. " +2024-01-10 20:39:47 | INFO | stdout | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32 +2024-01-10 20:39:50 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:39:50 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:48584 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:44296 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:56642 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:58994 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:58512 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | INFO: 172.17.8.150:41664 - "POST /worker_get_status HTTP/1.1" 200 OK +2024-01-10 20:39:50 | INFO | stdout | Running on local URL: http://localhost:7861 +2024-01-10 20:39:52 | INFO | stdout | Running on public URL: https://647288a5cc4c015ceb.gradio.live +2024-01-10 20:39:52 | INFO | stdout | +2024-01-10 20:39:52 | INFO | stdout | This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +2024-01-10 20:39:57 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/dropdown.py:176: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Dropdown(...)` instead of `return gr.Dropdown.update(...)`. +2024-01-10 20:39:57 | ERROR | stderr | warnings.warn( +2024-01-10 20:39:57 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/gradio/components/markdown.py:92: UserWarning: Using the update method is deprecated. Simply return a new object instead, e.g. `return gr.Markdown(...)` instead of `return gr.Markdown.update(...)`. +2024-01-10 20:39:57 | ERROR | stderr | warnings.warn( +2024-01-10 20:39:58 | ERROR | stderr | /h/tianleli/.conda/envs/imagen/lib/python3.9/asyncio/events.py:80: GradioUnusedKwargWarning: You have unused kwarg parameters in Button, please remove them: {'mode': 'static'} +2024-01-10 20:39:58 | ERROR | stderr | self._context.run(self._callback, *self._args) +2024-01-10 20:40:08 | INFO | stdout | INFO: 172.17.15.236:46634 - "POST /worker_generate_stream HTTP/1.1" 200 OK +2024-01-10 20:40:08 | INFO | diffusion_infer | prompt: a running dog +2024-01-10 20:40:09 | ERROR | stderr | 0%| | 0/50 [00:00: Failed to establish a new connection: [Errno 111] Connection refused +2024-01-11 09:32:37 | ERROR | stderr | +2024-01-11 09:32:37 | ERROR | stderr | The above exception was the direct cause of the following exception: +2024-01-11 09:32:37 | ERROR | stderr | +2024-01-11 09:32:37 | ERROR | stderr | Traceback (most recent call last): +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 486, in send +2024-01-11 09:32:37 | ERROR | stderr | resp = conn.urlopen( +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen +2024-01-11 09:32:37 | ERROR | stderr | retries = retries.increment( +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/urllib3/util/retry.py", line 515, in increment +2024-01-11 09:32:37 | ERROR | stderr | raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] +2024-01-11 09:32:37 | ERROR | stderr | urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='172.17.8.150', port=21001): Max retries exceeded with url: /register_worker (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')) +2024-01-11 09:32:37 | ERROR | stderr | +2024-01-11 09:32:37 | ERROR | stderr | During handling of the above exception, another exception occurred: +2024-01-11 09:32:37 | ERROR | stderr | +2024-01-11 09:32:37 | ERROR | stderr | Traceback (most recent call last): +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 197, in _run_module_as_main +2024-01-11 09:32:37 | ERROR | stderr | return _run_code(code, main_globals, None, +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/runpy.py", line 87, in _run_code +2024-01-11 09:32:37 | ERROR | stderr | exec(code, run_globals) +2024-01-11 09:32:37 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 390, in +2024-01-11 09:32:37 | ERROR | stderr | args, worker = create_model_worker() +2024-01-11 09:32:37 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 362, in create_model_worker +2024-01-11 09:32:37 | ERROR | stderr | worker = ModelWorker( +2024-01-11 09:32:37 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/model_worker.py", line 114, in __init__ +2024-01-11 09:32:37 | ERROR | stderr | self.init_heart_beat() +2024-01-11 09:32:37 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 79, in init_heart_beat +2024-01-11 09:32:37 | ERROR | stderr | self.register_to_controller() +2024-01-11 09:32:37 | ERROR | stderr | File "/fs01/home/tianleli/VisualArena/fastchat/serve/base_model_worker.py", line 96, in register_to_controller +2024-01-11 09:32:37 | ERROR | stderr | r = requests.post(url, json=data) +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 115, in post +2024-01-11 09:32:37 | ERROR | stderr | return request("post", url, data=data, json=json, **kwargs) +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/api.py", line 59, in request +2024-01-11 09:32:37 | ERROR | stderr | return session.request(method=method, url=url, **kwargs) +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 589, in request +2024-01-11 09:32:37 | ERROR | stderr | resp = self.send(prep, **send_kwargs) +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/sessions.py", line 703, in send +2024-01-11 09:32:37 | ERROR | stderr | r = adapter.send(request, **kwargs) +2024-01-11 09:32:37 | ERROR | stderr | File "/h/tianleli/.conda/envs/imagen/lib/python3.9/site-packages/requests/adapters.py", line 519, in send +2024-01-11 09:32:37 | ERROR | stderr | raise ConnectionError(e, request=request) +2024-01-11 09:32:37 | ERROR | stderr | requests.exceptions.ConnectionError: HTTPConnectionPool(host='172.17.8.150', port=21001): Max retries exceeded with url: /register_worker (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')) +2024-01-11 17:24:36 | INFO | stdout | Segment Anything Model (SAM) / GroundingDINO not installed. +2024-01-11 17:24:36 | INFO | stdout | +2024-01-11 17:24:40 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-11 17:24:40 | INFO | stdout | /fs01/home/tianleli/VisualArena +2024-01-11 17:24:41 | ERROR | stderr | Loading pipeline components...: 0%| | 0/6 [00:00 https://288918eb442b61a51d.gradio.live