[2024-09-19 18:45:26,922][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 18:45:26,925][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 18:48:22,919][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 18:48:22,922][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 18:49:51,412][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 18:49:51,415][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 18:58:36,280][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 18:58:36,280][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 18:58:36,282][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 18:58:36,282][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:00:18,345][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:00:18,345][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:00:18,345][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:00:18,348][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:00:18,348][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:00:18,348][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:06:37,319][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:06:37,319][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:06:37,319][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:06:37,319][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:06:37,322][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:06:37,322][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:06:37,322][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:06:37,322][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:07:21,727][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:07:21,730][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:07:54,313][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:07:54,313][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:07:54,317][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:07:54,317][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:11:51,338][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:11:51,338][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:11:51,338][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:11:51,342][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:11:51,342][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:11:51,342][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:12:04,149][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:12:04,149][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:12:04,149][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:12:04,149][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:12:04,153][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:12:04,153][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:12:04,153][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:12:04,153][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:16:20,720][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:16:20,723][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:16:56,610][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:16:56,613][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:18:21,078][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:18:21,081][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:18:55,360][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:18:55,360][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:18:55,362][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:18:55,362][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:19:00,843][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:19:00,843][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:19:00,843][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:19:00,845][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:19:00,845][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:19:00,845][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:20:28,761][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:20:28,764][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:21:11,769][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:21:11,773][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:21:43,626][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:21:43,629][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:27:21,566][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:27:21,566][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:27:21,569][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:27:21,569][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-19 19:27:35,878][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-19 19:27:35,881][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-20 02:15:00,062][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-20 02:15:00,065][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-20 02:30:32,505][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-20 02:30:32,509][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-20 02:37:17,022][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-20 02:37:17,022][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-20 02:37:17,024][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-20 02:37:17,024][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False
[2024-09-20 02:38:11,099][           solver.py][line:  88][    INFO] auto_denan disabled!
[2024-09-20 02:38:11,103][     solver_deter.py][line:  58][    INFO] deterministic mode, seed: 233, worker_rank: True,                                   cudnn_deterministic: False