Upload folder using huggingface_hub

.gitattributes

@@ -47,3 +47,28 @@ data/lean4_statement_translate/15k_state_problem_translation.json filter=lfs dif
 data/lean4_statement_translate/15k_state_problem_translation_statement_form.json filter=lfs diff=lfs merge=lfs -text
 data/lean4_statement_translate/15k_state_problem_translation_statementproof_inform.json filter=lfs diff=lfs merge=lfs -text
 data/test/math/train.jsonl filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/latest-run.10656/run-uzyiqb3v.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/latest-run.123918/run-itlu7dts.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240504_102712-1z84vjo1/run-1z84vjo1.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240505_142420-vx48r0mp/run-vx48r0mp.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240505_221021-1fhjldqc/run-1fhjldqc.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240506_155701-c0jyjrum/run-c0jyjrum.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240507_094958-uzyiqb3v/run-uzyiqb3v.wandb filter=lfs diff=lfs merge=lfs -text
+None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/run-itlu7dts.wandb filter=lfs diff=lfs merge=lfs -text
+data/continual_training/rft.json filter=lfs diff=lfs merge=lfs -text
+data/continual_training/rft_verifier_enhanced.json filter=lfs diff=lfs merge=lfs -text
+data/continual_training/verifier_enhanced.json filter=lfs diff=lfs merge=lfs -text
+data/lean4_basic/1k_test_filtered.jsonl filter=lfs diff=lfs merge=lfs -text
+data/lean4_random/15k_filtered.json filter=lfs diff=lfs merge=lfs -text
+data/lean4_random/1k_test_filtered.json filter=lfs diff=lfs merge=lfs -text
+data/lean4_random/5k_first_filtered.json filter=lfs diff=lfs merge=lfs -text
+data/lean4_random/5k_third_filtered.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/0.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/1.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/2.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/3.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/4.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/5.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/6.json filter=lfs diff=lfs merge=lfs -text
+generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/7.json filter=lfs diff=lfs merge=lfs -text
+user._COPYING_/chijx/test_file_1GB._COPYING_ filter=lfs diff=lfs merge=lfs -text

None/verifier-reproduce/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Configure stats pid to 123918
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/mariana/auto-info/wandb/settings
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'auto-info/PSV/train_psv.py', 'program': '/opt/tiger/mariana/auto-info/PSV/train_psv.py'}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/logs/debug.log
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/logs/debug-internal.log
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():518] calling init triggers
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():571] starting backend
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():575] setting up manager
+2024-05-07 17:25:51,800 INFO    MainThread:123918 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-07 17:25:51,802 INFO    MainThread:123918 [wandb_init.py:init():582] backend started and connected
+2024-05-07 17:25:51,805 INFO    MainThread:123918 [wandb_init.py:init():660] updated telemetry
+2024-05-07 17:25:52,381 INFO    MainThread:123918 [wandb_init.py:init():695] communicating run to backend with 60 second timeout
+2024-05-07 17:25:53,306 INFO    MainThread:123918 [wandb_run.py:_on_init():1998] communicating current version
+2024-05-07 17:25:58,306 INFO    MainThread:123918 [wandb_run.py:_on_init():2002] got version response None
+2024-05-07 17:25:58,306 INFO    MainThread:123918 [wandb_init.py:init():730] starting run threads in backend
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_console_start():1979] atexit reg
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1837] redirect: SettingsConsole.WRAP_RAW
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1902] Wrapping output streams.
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1924] Redirects installed.
+2024-05-07 17:25:58,472 INFO    MainThread:123918 [wandb_init.py:init():767] run started, returning control to user process
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_finish():1746] finishing run /verifier-reproduce/itlu7dts
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_atexit_cleanup():1948] got exitcode: 0
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_restore():1931] restore
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_restore():1937] restore done
+2024-05-07 20:28:07,612 INFO    MainThread:123918 [wandb_run.py:_footer_history_summary_info():3391] rendering history
+2024-05-07 20:28:07,612 INFO    MainThread:123918 [wandb_run.py:_footer_history_summary_info():3423] rendering summary
+2024-05-07 20:28:07,613 INFO    MainThread:123918 [wandb_run.py:_footer_sync_info():3349] logging synced files

None/verifier-reproduce/wandb/latest-run.10656/files/config.yaml

@@ -0,0 +1 @@
+wandb_version: 1

None/verifier-reproduce/wandb/latest-run.10656/files/output.log

@@ -0,0 +1,748 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 729/729 [3:01:32<00:00, 14.94s/it, epoch=0, llm_loss=0.495, loss=6.37, step=728, v_loss=5.88]
++ [Save] Save model and tokenizer to: /opt/tiger/models/lean4_random_15k_all-sample10-osv-gt2
++ [Save] Save training_args to: /opt/tiger/models/lean4_random_15k_all-sample10-osv-gt2/training_args.bin

None/verifier-reproduce/wandb/latest-run.10656/files/requirements.txt

@@ -0,0 +1,316 @@
+absl-py==0.15.0
+accelerate==0.30.0
+addict==2.4.0
+aiofiles==22.1.0
+aiosqlite==0.20.0
+annotated-types==0.6.0
+anyio==4.3.0
+apex==0.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+babel==2.15.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+byted-dataloader==0.4.4
+byted-encrypted-hdfs==0.5.5
+byted-iceberg==0.2.112
+byted-lafka-internal==1.4.16rc1
+byted-remote-ikernel==0.4.8
+byted-torch-monitor==0.0.1
+byted-torch==2.1.0.post2
+byted-wandb==0.13.72
+bytedance-context==0.7.1
+bytedance-metrics==0.5.1
+bytedance.modelhub==0.0.63
+bytedance.servicediscovery==0.1.2
+bytedbackgrounds==0.0.6
+byteddatabus==1.0.6
+byteddps==0.1.2
+bytedenv==0.6.2
+bytedeuler==0.42.2
+bytedfeather==0.3.5
+bytedkafka==0.2.9
+bytedkms==3.2.9
+bytedkmsv2==0.9.35
+bytedlego-ops==1.0.0.218
+bytedlogger==0.15.1
+bytedlogid==0.2.1
+bytedmemfd==0.2
+bytedmetrics==0.9.1
+bytedmoxing==0.2.0
+bytedpymongo==2.0.5
+bytedrh2==1.18.9
+bytedservicediscovery==0.17.4
+bytedsinfmetacenter==1.4.3
+bytedtcc==1.4.2
+bytedtitan==1.0.0
+bytedtos==1.1.16
+bytedtrace==0.3.0
+bytedztijwthelper==0.0.22
+bytedztispiffe==0.0.11
+cachetools==5.3.3
+certifi==2022.12.7
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.1.1
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.0
+cramjam==2.8.3
+crcmod==1.7
+cryptography==38.0.4
+cycler==0.12.1
+dbus-python==1.2.16
+debugpy==1.8.1
+decorator==5.1.1
+deepspeed==0.14.0
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.8
+distlib==0.3.8
+distro-info==1.0
+dm-tree==0.1.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring-parser==0.16
+einops==0.7.0
+entrypoints==0.4
+exceptiongroup==1.2.1
+executing==2.0.1
+fastjsonschema==2.19.1
+filelock==3.13.3
+flash-attn==2.5.6
+flatbuffers==24.3.25
+fonttools==4.50.0
+fqdn==1.5.1
+fsspec==2024.3.1
+fvcore==0.1.5.post20221221
+gast==0.5.4
+gevent==22.10.2
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.29.0
+google-pasta==0.2.0
+greenlet==3.0.3
+grpcio==1.62.1
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+huggingface-hub==0.22.2
+idna==3.4
+importlib-metadata==7.1.0
+importlib-resources==6.4.0
+iniconfig==2.0.0
+iopath==0.1.10
+iotop==0.6
+ipaddress==1.0.23
+ipykernel==6.29.4
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.2
+isoduration==20.11.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.25
+jsonargparse==4.14.0
+jsonpath-ng==1.5.3
+jsonpointer==2.4
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-client==7.0.0
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyter-events==0.10.0
+jupyter-server-fileid==0.9.2
+jupyter-server-terminals==0.5.3
+jupyter-server-ydoc==0.8.0
+jupyter-server==2.14.0
+jupyter-ydoc==0.2.5
+jupyter==1.0.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.27.1
+jupyterlab-widgets==3.0.10
+jupyterlab==3.6.4
+keras-preprocessing==1.1.2
+keras==2.8.0
+kiwisolver==1.4.5
+lazy-loader==0.3
+libclang==18.1.1
+librosa==0.10.1
+llvmlite==0.42.0
+markdown-it-py==3.0.0
+markdown==3.6
+markupsafe==2.1.2
+matplotlib-inline==0.1.7
+matplotlib==3.8.3
+mdurl==0.1.2
+mistune==3.0.2
+mmhash3==3.0.1
+mpmath==1.2.1
+msgpack==1.0.8
+nbclassic==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+notebook-shim==0.2.4
+notebook==6.5.7
+numba==0.59.1
+numpy==1.25.2
+nvidia-dali-cuda120==1.36.0
+nvidia-nvimgcodec-cu12==0.2.0.7
+oauthlib==3.2.2
+onnx==1.16.0
+opt-einsum==3.3.0
+overrides==7.7.0
+packaging==24.0
+pandas==1.3.5
+pandocfilters==1.5.1
+parso==0.8.4
+pathlib2==2.3.7.post1
+pathtools==0.1.2
+pexpect==4.8.0
+pillow==9.3.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+pooch==1.8.1
+portalocker==2.8.2
+prettytable==3.10.0
+prometheus-client==0.20.0
+promise==2.3
+prompt-toolkit==3.0.43
+protobuf==3.20.0
+psutil==5.9.8
+ptflops==0.7.2.2
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py==1.11.0
+pyarrow==11.0.0
+pyasn1-modules==0.4.0
+pyasn1==0.6.0
+pybind11==2.12.0
+pycparser==2.22
+pycryptodomex==3.20.0
+pycurl==7.43.0.6
+pydantic-core==2.16.3
+pydantic==1.10.14
+pygments==2.17.2
+pygobject==3.38.0
+pyjwt==2.8.0
+pynvml==11.5.0
+pyopenssl==22.1.0
+pyparsing==3.1.2
+pypeg2==2.15.2
+pytest==6.2.5
+python-apt==2.2.1
+python-consul==1.1.0
+python-dateutil==2.9.0.post0
+python-engineio==4.9.0
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-snappy==0.7.1
+python-socketio==5.11.2
+pytz==2024.1
+pyyaml==5.4.1
+pyzmq==26.0.3
+qtconsole==5.5.2
+qtpy==2.4.1
+redis==4.5.5
+referencing==0.35.1
+regex==2023.12.25
+requests-oauthlib==2.0.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.1
+rsa==4.9
+safetensors==0.4.2
+schedule==1.2.1
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+send2trash==1.8.3
+sentencepiece==0.2.0
+sentry-sdk==1.44.0
+setproctitle==1.3.3
+setuptools==68.1.2
+shortuuid==1.0.13
+simple-websocket==1.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+stack-data==0.6.3
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.8.0
+tensorflow-cpu==2.8.0
+tensorflow-io-gcs-filesystem==0.25.0
+tensorflow-io==0.25.0
+termcolor==2.4.0
+terminado==0.18.1
+terminaltables==3.1.10
+text-cutter==2.3.0
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.4.0
+thrift2pyi==1.0.3
+thriftpy2==0.4.20
+timm==0.9.16
+tinycss2==1.3.0
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+torch==2.1.0+cu121
+torchaudio==2.1.0+cu121
+torchlibrosa==0.1.0
+torchvision==0.16.0+cu121
+tornado==6.4
+tox==3.28.0
+tqdm==4.48.2
+traitlets==5.14.3
+transformers==4.39.2
+triton==2.1.0
+types-python-dateutil==2.9.0.20240316
+typing-extensions==4.10.0
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==1.26.13
+virtualenv==20.25.1
+watchdog==4.0.0
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.2
+wheel==0.41.2
+widgetsnbextension==4.0.10
+wrapt==1.16.0
+wsproto==1.2.0
+y-py==0.6.2
+yacs==0.1.8
+yapf==0.40.2
+ypy-websocket==0.8.4
+zipp==3.18.1
+zope.event==5.0
+zope.interface==6.2
+zstandard==0.19.0

None/verifier-reproduce/wandb/latest-run.10656/files/wandb-metadata.json

@@ -0,0 +1,1032 @@
+{
+    "os": "Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31",
+    "python": "3.9.2",
+    "heartbeatAt": "2024-05-07T01:50:05.016076",
+    "startedAt": "2024-05-07T01:49:58.414011",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "/opt/tiger/models/lean4_random_15k_all/checkpoint-236/",
+        "--data_dir",
+        "../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl",
+        "--data_id",
+        "lean4test",
+        "--target_set",
+        "train",
+        "--save_dir",
+        "/opt/tiger/models/lean4_random_15k_all-sample10-osv-gt2",
+        "--generator_id",
+        "lean4_random_15k_all",
+        "--verifier_id",
+        "osv",
+        "--dedup",
+        "True",
+        "--per_problem_sampling_solution",
+        "10",
+        "--loss_level",
+        "token",
+        "--loss_on_llm",
+        "True",
+        "--num_train_epoches",
+        "1",
+        "--eval_steps",
+        "10000",
+        "--per_device_train_batch_size",
+        "16",
+        "--per_device_eval_batch_size",
+        "64",
+        "--gradient_accumulation_steps",
+        "4",
+        "--gradient_checkpointing",
+        "True",
+        "--learning_rate",
+        "2e-6",
+        "--weight_decay",
+        "0",
+        "--save_steps",
+        "1000",
+        "--lr_scheduler_type",
+        "linear",
+        "--warmup_steps",
+        "0",
+        "--save_epoches",
+        "1",
+        "--save_best",
+        "False",
+        "--save_total_limit",
+        "0",
+        "--logging_dir",
+        "None",
+        "--logging_steps",
+        "1",
+        "--easy",
+        "True",
+        "--seed",
+        "42"
+    ],
+    "state": "running",
+    "program": "/opt/tiger/mariana/auto-info/PSV/train_psv.py",
+    "codePath": "auto-info/PSV/train_psv.py",
+    "git": {
+        "remote": "[email protected]:seed/mariana.git",
+        "commit": "ce8cbcbc8313f5085e52656d8bbf2e60e3c0d785"
+    },
+    "email": "[email protected]",
+    "root": "/opt/tiger/mariana",
+    "host": "n122-143-013",
+    "username": "tiger",
+    "executable": "/usr/bin/python3",
+    "cpu_count": 90,
+    "cpu_count_logical": 180,
+    "cpu_freq": {
+        "current": 2600.0250000000083,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "total": 3519.748878479004,
+        "used": 296.1867370605469
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2932.9151611328125
+    }
+}

None/verifier-reproduce/wandb/latest-run.10656/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss": 2.069721221923828, "v_loss": 1.7109375, "llm_loss": 0.3587838411331177, "lr": 0.0, "_timestamp": 1715057487.8088303, "_runtime": 10889.386602163315, "_step": 727, "_wandb": {"runtime": 10921}}

None/verifier-reproduce/wandb/latest-run.10656/logs/debug.log

@@ -0,0 +1,31 @@
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_setup.py:_flush():68] Configure stats pid to 10656
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/mariana/auto-info/wandb/settings
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'auto-info/PSV/train_psv.py', 'program': '/opt/tiger/mariana/auto-info/PSV/train_psv.py'}
+2024-05-07 09:49:58,415 INFO    MainThread:10656 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240507_094958-uzyiqb3v/logs/debug.log
+2024-05-07 09:49:58,416 INFO    MainThread:10656 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240507_094958-uzyiqb3v/logs/debug-internal.log
+2024-05-07 09:49:58,416 INFO    MainThread:10656 [wandb_init.py:init():518] calling init triggers
+2024-05-07 09:49:58,416 INFO    MainThread:10656 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-07 09:49:58,416 INFO    MainThread:10656 [wandb_init.py:init():571] starting backend
+2024-05-07 09:49:58,416 INFO    MainThread:10656 [wandb_init.py:init():575] setting up manager
+2024-05-07 09:49:58,420 INFO    MainThread:10656 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-07 09:49:58,422 INFO    MainThread:10656 [wandb_init.py:init():582] backend started and connected
+2024-05-07 09:49:58,425 INFO    MainThread:10656 [wandb_init.py:init():660] updated telemetry
+2024-05-07 09:49:58,979 INFO    MainThread:10656 [wandb_init.py:init():695] communicating run to backend with 60 second timeout
+2024-05-07 09:49:59,911 INFO    MainThread:10656 [wandb_run.py:_on_init():1998] communicating current version
+2024-05-07 09:50:04,911 INFO    MainThread:10656 [wandb_run.py:_on_init():2002] got version response None
+2024-05-07 09:50:04,911 INFO    MainThread:10656 [wandb_init.py:init():730] starting run threads in backend
+2024-05-07 09:50:05,091 INFO    MainThread:10656 [wandb_run.py:_console_start():1979] atexit reg
+2024-05-07 09:50:05,092 INFO    MainThread:10656 [wandb_run.py:_redirect():1837] redirect: SettingsConsole.WRAP_RAW
+2024-05-07 09:50:05,092 INFO    MainThread:10656 [wandb_run.py:_redirect():1902] Wrapping output streams.
+2024-05-07 09:50:05,092 INFO    MainThread:10656 [wandb_run.py:_redirect():1924] Redirects installed.
+2024-05-07 09:50:05,092 INFO    MainThread:10656 [wandb_init.py:init():767] run started, returning control to user process
+2024-05-07 12:52:06,633 INFO    MainThread:10656 [wandb_run.py:_finish():1746] finishing run /verifier-reproduce/uzyiqb3v
+2024-05-07 12:52:06,633 INFO    MainThread:10656 [wandb_run.py:_atexit_cleanup():1948] got exitcode: 0
+2024-05-07 12:52:06,634 INFO    MainThread:10656 [wandb_run.py:_restore():1931] restore
+2024-05-07 12:52:06,634 INFO    MainThread:10656 [wandb_run.py:_restore():1937] restore done
+2024-05-07 12:52:10,295 INFO    MainThread:10656 [wandb_run.py:_footer_history_summary_info():3391] rendering history
+2024-05-07 12:52:10,296 INFO    MainThread:10656 [wandb_run.py:_footer_history_summary_info():3423] rendering summary
+2024-05-07 12:52:10,296 INFO    MainThread:10656 [wandb_run.py:_footer_sync_info():3349] logging synced files

None/verifier-reproduce/wandb/latest-run.10656/run-uzyiqb3v.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11f4f8b659fa4bacd6b622cd078ab8f242962a0f2d052116f6ed66e8bcf9a316
+size 3228495

None/verifier-reproduce/wandb/latest-run.10656/tracking_info.json

@@ -0,0 +1 @@
+{"project_id": "", "run_id": "uzyiqb3v", "run_link": "https://ml.bytedance.net/experiment/tracking/detail?Id=project_20240412_f2edc88c&selectedTrial=uzyiqb3v"}

None/verifier-reproduce/wandb/latest-run.123918/files/config.yaml

@@ -0,0 +1 @@
+wandb_version: 1

None/verifier-reproduce/wandb/latest-run.123918/files/output.log

@@ -0,0 +1,751 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 729/729 [3:01:36<00:00, 14.95s/it, epoch=0, llm_loss=0.562, loss=4.2, step=728, v_loss=3.64]
++ [Save] Save model and tokenizer to: /opt/tiger/models/lean4_random_15k_all-sample10-psv-0.5
++ [Save] Save training_args to: /opt/tiger/models/lean4_random_15k_all-sample10-psv-0.5/training_args.bin

None/verifier-reproduce/wandb/latest-run.123918/files/requirements.txt

@@ -0,0 +1,316 @@
+absl-py==0.15.0
+accelerate==0.30.0
+addict==2.4.0
+aiofiles==22.1.0
+aiosqlite==0.20.0
+annotated-types==0.6.0
+anyio==4.3.0
+apex==0.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+babel==2.15.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+byted-dataloader==0.4.4
+byted-encrypted-hdfs==0.5.5
+byted-iceberg==0.2.112
+byted-lafka-internal==1.4.16rc1
+byted-remote-ikernel==0.4.8
+byted-torch-monitor==0.0.1
+byted-torch==2.1.0.post2
+byted-wandb==0.13.72
+bytedance-context==0.7.1
+bytedance-metrics==0.5.1
+bytedance.modelhub==0.0.63
+bytedance.servicediscovery==0.1.2
+bytedbackgrounds==0.0.6
+byteddatabus==1.0.6
+byteddps==0.1.2
+bytedenv==0.6.2
+bytedeuler==0.42.2
+bytedfeather==0.3.5
+bytedkafka==0.2.9
+bytedkms==3.2.9
+bytedkmsv2==0.9.35
+bytedlego-ops==1.0.0.218
+bytedlogger==0.15.1
+bytedlogid==0.2.1
+bytedmemfd==0.2
+bytedmetrics==0.9.1
+bytedmoxing==0.2.0
+bytedpymongo==2.0.5
+bytedrh2==1.18.9
+bytedservicediscovery==0.17.4
+bytedsinfmetacenter==1.4.3
+bytedtcc==1.4.2
+bytedtitan==1.0.0
+bytedtos==1.1.16
+bytedtrace==0.3.0
+bytedztijwthelper==0.0.22
+bytedztispiffe==0.0.11
+cachetools==5.3.3
+certifi==2022.12.7
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.1.1
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.0
+cramjam==2.8.3
+crcmod==1.7
+cryptography==38.0.4
+cycler==0.12.1
+dbus-python==1.2.16
+debugpy==1.8.1
+decorator==5.1.1
+deepspeed==0.14.0
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.8
+distlib==0.3.8
+distro-info==1.0
+dm-tree==0.1.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring-parser==0.16
+einops==0.7.0
+entrypoints==0.4
+exceptiongroup==1.2.1
+executing==2.0.1
+fastjsonschema==2.19.1
+filelock==3.13.3
+flash-attn==2.5.6
+flatbuffers==24.3.25
+fonttools==4.50.0
+fqdn==1.5.1
+fsspec==2024.3.1
+fvcore==0.1.5.post20221221
+gast==0.5.4
+gevent==22.10.2
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.29.0
+google-pasta==0.2.0
+greenlet==3.0.3
+grpcio==1.62.1
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+huggingface-hub==0.22.2
+idna==3.4
+importlib-metadata==7.1.0
+importlib-resources==6.4.0
+iniconfig==2.0.0
+iopath==0.1.10
+iotop==0.6
+ipaddress==1.0.23
+ipykernel==6.29.4
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.2
+isoduration==20.11.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.25
+jsonargparse==4.14.0
+jsonpath-ng==1.5.3
+jsonpointer==2.4
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-client==7.0.0
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyter-events==0.10.0
+jupyter-server-fileid==0.9.2
+jupyter-server-terminals==0.5.3
+jupyter-server-ydoc==0.8.0
+jupyter-server==2.14.0
+jupyter-ydoc==0.2.5
+jupyter==1.0.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.27.1
+jupyterlab-widgets==3.0.10
+jupyterlab==3.6.4
+keras-preprocessing==1.1.2
+keras==2.8.0
+kiwisolver==1.4.5
+lazy-loader==0.3
+libclang==18.1.1
+librosa==0.10.1
+llvmlite==0.42.0
+markdown-it-py==3.0.0
+markdown==3.6
+markupsafe==2.1.2
+matplotlib-inline==0.1.7
+matplotlib==3.8.3
+mdurl==0.1.2
+mistune==3.0.2
+mmhash3==3.0.1
+mpmath==1.2.1
+msgpack==1.0.8
+nbclassic==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+notebook-shim==0.2.4
+notebook==6.5.7
+numba==0.59.1
+numpy==1.25.2
+nvidia-dali-cuda120==1.36.0
+nvidia-nvimgcodec-cu12==0.2.0.7
+oauthlib==3.2.2
+onnx==1.16.0
+opt-einsum==3.3.0
+overrides==7.7.0
+packaging==24.0
+pandas==1.3.5
+pandocfilters==1.5.1
+parso==0.8.4
+pathlib2==2.3.7.post1
+pathtools==0.1.2
+pexpect==4.8.0
+pillow==9.3.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+pooch==1.8.1
+portalocker==2.8.2
+prettytable==3.10.0
+prometheus-client==0.20.0
+promise==2.3
+prompt-toolkit==3.0.43
+protobuf==3.20.0
+psutil==5.9.8
+ptflops==0.7.2.2
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py==1.11.0
+pyarrow==11.0.0
+pyasn1-modules==0.4.0
+pyasn1==0.6.0
+pybind11==2.12.0
+pycparser==2.22
+pycryptodomex==3.20.0
+pycurl==7.43.0.6
+pydantic-core==2.16.3
+pydantic==1.10.14
+pygments==2.17.2
+pygobject==3.38.0
+pyjwt==2.8.0
+pynvml==11.5.0
+pyopenssl==22.1.0
+pyparsing==3.1.2
+pypeg2==2.15.2
+pytest==6.2.5
+python-apt==2.2.1
+python-consul==1.1.0
+python-dateutil==2.9.0.post0
+python-engineio==4.9.0
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-snappy==0.7.1
+python-socketio==5.11.2
+pytz==2024.1
+pyyaml==5.4.1
+pyzmq==26.0.3
+qtconsole==5.5.2
+qtpy==2.4.1
+redis==4.5.5
+referencing==0.35.1
+regex==2023.12.25
+requests-oauthlib==2.0.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.1
+rsa==4.9
+safetensors==0.4.2
+schedule==1.2.1
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+send2trash==1.8.3
+sentencepiece==0.2.0
+sentry-sdk==1.44.0
+setproctitle==1.3.3
+setuptools==68.1.2
+shortuuid==1.0.13
+simple-websocket==1.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+stack-data==0.6.3
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.8.0
+tensorflow-cpu==2.8.0
+tensorflow-io-gcs-filesystem==0.25.0
+tensorflow-io==0.25.0
+termcolor==2.4.0
+terminado==0.18.1
+terminaltables==3.1.10
+text-cutter==2.3.0
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.4.0
+thrift2pyi==1.0.3
+thriftpy2==0.4.20
+timm==0.9.16
+tinycss2==1.3.0
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+torch==2.1.0+cu121
+torchaudio==2.1.0+cu121
+torchlibrosa==0.1.0
+torchvision==0.16.0+cu121
+tornado==6.4
+tox==3.28.0
+tqdm==4.48.2
+traitlets==5.14.3
+transformers==4.39.2
+triton==2.1.0
+types-python-dateutil==2.9.0.20240316
+typing-extensions==4.10.0
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==1.26.13
+virtualenv==20.25.1
+watchdog==4.0.0
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.2
+wheel==0.41.2
+widgetsnbextension==4.0.10
+wrapt==1.16.0
+wsproto==1.2.0
+y-py==0.6.2
+yacs==0.1.8
+yapf==0.40.2
+ypy-websocket==0.8.4
+zipp==3.18.1
+zope.event==5.0
+zope.interface==6.2
+zstandard==0.19.0

None/verifier-reproduce/wandb/latest-run.123918/files/wandb-metadata.json

@@ -0,0 +1,1034 @@
+{
+    "os": "Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31",
+    "python": "3.9.2",
+    "heartbeatAt": "2024-05-07T09:25:58.411901",
+    "startedAt": "2024-05-07T09:25:51.794582",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "/opt/tiger/models/lean4_random_15k_all/checkpoint-236/",
+        "--data_dir",
+        "../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl",
+        "--data_id",
+        "lean4test",
+        "--target_set",
+        "train",
+        "--save_dir",
+        "/opt/tiger/models/lean4_random_15k_all-sample10-psv-0.5",
+        "--generator_id",
+        "lean4_random_15k_all",
+        "--verifier_id",
+        "psv-0.5",
+        "--dedup",
+        "True",
+        "--per_problem_sampling_solution",
+        "10",
+        "--loss_level",
+        "token",
+        "--loss_on_llm",
+        "True",
+        "--num_train_epoches",
+        "1",
+        "--eval_steps",
+        "10000",
+        "--per_device_train_batch_size",
+        "16",
+        "--per_device_eval_batch_size",
+        "64",
+        "--gradient_accumulation_steps",
+        "4",
+        "--gradient_checkpointing",
+        "True",
+        "--learning_rate",
+        "2e-6",
+        "--weight_decay",
+        "0",
+        "--save_steps",
+        "1000",
+        "--lr_scheduler_type",
+        "linear",
+        "--warmup_steps",
+        "0",
+        "--save_epoches",
+        "1",
+        "--save_best",
+        "False",
+        "--save_total_limit",
+        "0",
+        "--logging_dir",
+        "None",
+        "--logging_steps",
+        "1",
+        "--easy",
+        "True",
+        "--process",
+        "True",
+        "--seed",
+        "42"
+    ],
+    "state": "running",
+    "program": "/opt/tiger/mariana/auto-info/PSV/train_psv.py",
+    "codePath": "auto-info/PSV/train_psv.py",
+    "git": {
+        "remote": "[email protected]:seed/mariana.git",
+        "commit": "ce8cbcbc8313f5085e52656d8bbf2e60e3c0d785"
+    },
+    "email": "[email protected]",
+    "root": "/opt/tiger/mariana",
+    "host": "n122-143-013",
+    "username": "tiger",
+    "executable": "/usr/bin/python3",
+    "cpu_count": 90,
+    "cpu_count_logical": 180,
+    "cpu_freq": {
+        "current": 2600.0250000000083,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.025,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "total": 3519.748878479004,
+        "used": 299.1645317077637
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2932.9151611328125
+    }
+}

None/verifier-reproduce/wandb/latest-run.123918/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss": 1.3389185667037964, "v_loss": 0.875, "llm_loss": 0.4639185965061188, "lr": 0.0, "_timestamp": 1715084844.8470807, "_runtime": 10893.044775724411, "_step": 727, "_wandb": {"runtime": 10925}}

None/verifier-reproduce/wandb/latest-run.123918/logs/debug.log

@@ -0,0 +1,31 @@
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Configure stats pid to 123918
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/mariana/auto-info/wandb/settings
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'auto-info/PSV/train_psv.py', 'program': '/opt/tiger/mariana/auto-info/PSV/train_psv.py'}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/logs/debug.log
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240507_172551-itlu7dts/logs/debug-internal.log
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():518] calling init triggers
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():571] starting backend
+2024-05-07 17:25:51,796 INFO    MainThread:123918 [wandb_init.py:init():575] setting up manager
+2024-05-07 17:25:51,800 INFO    MainThread:123918 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-07 17:25:51,802 INFO    MainThread:123918 [wandb_init.py:init():582] backend started and connected
+2024-05-07 17:25:51,805 INFO    MainThread:123918 [wandb_init.py:init():660] updated telemetry
+2024-05-07 17:25:52,381 INFO    MainThread:123918 [wandb_init.py:init():695] communicating run to backend with 60 second timeout
+2024-05-07 17:25:53,306 INFO    MainThread:123918 [wandb_run.py:_on_init():1998] communicating current version
+2024-05-07 17:25:58,306 INFO    MainThread:123918 [wandb_run.py:_on_init():2002] got version response None
+2024-05-07 17:25:58,306 INFO    MainThread:123918 [wandb_init.py:init():730] starting run threads in backend
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_console_start():1979] atexit reg
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1837] redirect: SettingsConsole.WRAP_RAW
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1902] Wrapping output streams.
+2024-05-07 17:25:58,471 INFO    MainThread:123918 [wandb_run.py:_redirect():1924] Redirects installed.
+2024-05-07 17:25:58,472 INFO    MainThread:123918 [wandb_init.py:init():767] run started, returning control to user process
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_finish():1746] finishing run /verifier-reproduce/itlu7dts
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_atexit_cleanup():1948] got exitcode: 0
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_restore():1931] restore
+2024-05-07 20:28:03,757 INFO    MainThread:123918 [wandb_run.py:_restore():1937] restore done
+2024-05-07 20:28:07,612 INFO    MainThread:123918 [wandb_run.py:_footer_history_summary_info():3391] rendering history
+2024-05-07 20:28:07,612 INFO    MainThread:123918 [wandb_run.py:_footer_history_summary_info():3423] rendering summary
+2024-05-07 20:28:07,613 INFO    MainThread:123918 [wandb_run.py:_footer_sync_info():3349] logging synced files

None/verifier-reproduce/wandb/latest-run.123918/run-itlu7dts.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c72c26beccc9352b61579e45fa02e4e85c448ecdf4a6cc0bfd0dd41bcf10d0bd
+size 3233805

None/verifier-reproduce/wandb/latest-run.123918/tracking_info.json

@@ -0,0 +1 @@
+{"project_id": "", "run_id": "itlu7dts", "run_link": "https://ml.bytedance.net/experiment/tracking/detail?Id=project_20240412_f2edc88c&selectedTrial=itlu7dts"}

None/verifier-reproduce/wandb/latest-run/files/config.yaml

@@ -0,0 +1 @@
+wandb_version: 1

None/verifier-reproduce/wandb/latest-run/files/output.log

@@ -0,0 +1,6 @@
+
+
+
+
+
+

None/verifier-reproduce/wandb/latest-run/files/requirements.txt

@@ -0,0 +1,346 @@
+absl-py==0.15.0
+accelerate==0.29.3
+addict==2.4.0
+aiofiles==22.1.0
+aioprometheus==23.12.0
+aiosignal==1.3.1
+aiosqlite==0.20.0
+annotated-types==0.6.0
+anyio==4.3.0
+apex==0.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+babel==2.14.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+byted-dataloader==0.4.4
+byted-encrypted-hdfs==0.5.5
+byted-iceberg==0.2.112
+byted-lafka-internal==1.4.16rc1
+byted-remote-ikernel==0.4.8
+byted-torch-monitor==0.0.1
+byted-torch==2.1.0.post2
+byted-wandb==0.13.72
+bytedance-context==0.7.1
+bytedance-metrics==0.5.1
+bytedance.modelhub==0.0.63
+bytedance.servicediscovery==0.1.2
+bytedbackgrounds==0.0.6
+byteddatabus==1.0.6
+byteddps==0.1.2
+bytedenv==0.6.2
+bytedeuler==0.42.2
+bytedfeather==0.3.5
+bytedkafka==0.2.9
+bytedkms==3.2.9
+bytedkmsv2==0.9.35
+bytedlego-ops==1.0.0.218
+bytedlogger==0.15.1
+bytedlogid==0.2.1
+bytedmemfd==0.2
+bytedmetrics==0.9.1
+bytedmoxing==0.2.0
+bytedpymongo==2.0.5
+bytedrh2==1.18.9
+bytedservicediscovery==0.17.4
+bytedsinfmetacenter==1.4.3
+bytedtcc==1.4.2
+bytedtitan==1.0.0
+bytedtos==1.1.16
+bytedtrace==0.3.0
+bytedztijwthelper==0.0.22
+bytedztispiffe==0.0.11
+cachetools==5.3.3
+certifi==2022.12.7
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.1.1
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.0
+cramjam==2.8.3
+crcmod==1.7
+cryptography==38.0.4
+cupy-cuda12x==12.1.0
+cycler==0.12.1
+dbus-python==1.2.16
+debugpy==1.8.1
+decorator==5.1.1
+deepspeed==0.14.0
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.8
+distlib==0.3.8
+distro-info==1.0
+dm-tree==0.1.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring-parser==0.16
+einops==0.7.0
+entrypoints==0.4
+exceptiongroup==1.2.1
+executing==2.0.1
+fastapi==0.110.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.3
+flash-attn==2.5.6
+flatbuffers==24.3.25
+fonttools==4.50.0
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+fvcore==0.1.5.post20221221
+gast==0.5.4
+gevent==22.10.2
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.29.0
+google-pasta==0.2.0
+greenlet==3.0.3
+grpcio==1.62.1
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+httptools==0.6.1
+huggingface-hub==0.22.2
+idna==3.4
+importlib-metadata==7.1.0
+importlib-resources==6.4.0
+iniconfig==2.0.0
+iopath==0.1.10
+iotop==0.6
+ipaddress==1.0.23
+ipykernel==6.29.4
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.2
+isoduration==20.11.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.25
+jsonargparse==4.14.0
+jsonpath-ng==1.5.3
+jsonpointer==2.4
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-client==7.0.0
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyter-events==0.10.0
+jupyter-server-fileid==0.9.2
+jupyter-server-terminals==0.5.3
+jupyter-server-ydoc==0.8.0
+jupyter-server==2.14.0
+jupyter-ydoc==0.2.5
+jupyter==1.0.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.27.1
+jupyterlab-widgets==3.0.10
+jupyterlab==3.6.4
+keras-preprocessing==1.1.2
+keras==2.8.0
+kiwisolver==1.4.5
+lazy-loader==0.3
+libclang==18.1.1
+librosa==0.10.1
+llvmlite==0.42.0
+markdown-it-py==3.0.0
+markdown==3.6
+markupsafe==2.1.2
+matplotlib-inline==0.1.7
+matplotlib==3.8.3
+mdurl==0.1.2
+mistune==3.0.2
+mmhash3==3.0.1
+mpmath==1.2.1
+msgpack==1.0.8
+nbclassic==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+notebook-shim==0.2.4
+notebook==6.5.7
+numba==0.59.1
+numpy==1.25.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-dali-cuda120==1.36.0
+nvidia-nccl-cu12==2.18.1
+nvidia-nvimgcodec-cu12==0.2.0.7
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+onnx==1.16.0
+opt-einsum==3.3.0
+orjson==3.10.2
+overrides==7.7.0
+packaging==24.0
+pandas==1.3.5
+pandocfilters==1.5.1
+parso==0.8.4
+pathlib2==2.3.7.post1
+pathtools==0.1.2
+pexpect==4.8.0
+pillow==9.3.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+pooch==1.8.1
+portalocker==2.8.2
+prettytable==3.10.0
+prometheus-client==0.20.0
+promise==2.3
+prompt-toolkit==3.0.43
+protobuf==3.20.0
+psutil==5.9.8
+ptflops==0.7.2.2
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py==1.11.0
+pyarrow==11.0.0
+pyasn1-modules==0.4.0
+pyasn1==0.6.0
+pybind11==2.12.0
+pycparser==2.22
+pycryptodomex==3.20.0
+pycurl==7.43.0.6
+pydantic-core==2.18.2
+pydantic==2.7.1
+pygments==2.17.2
+pygobject==3.38.0
+pyjwt==2.8.0
+pynvml==11.5.0
+pyopenssl==22.1.0
+pyparsing==3.1.2
+pypeg2==2.15.2
+pytest==6.2.5
+python-apt==2.2.1
+python-consul==1.1.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-engineio==4.9.0
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-snappy==0.7.1
+python-socketio==5.11.2
+pytz==2024.1
+pyyaml==5.4.1
+pyzmq==26.0.3
+qtconsole==5.5.1
+qtpy==2.4.1
+quantile-python==1.1
+ray==2.20.0
+redis==4.5.5
+referencing==0.35.1
+regex==2023.12.25
+requests-oauthlib==2.0.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.0
+rsa==4.9
+safetensors==0.4.2
+schedule==1.2.1
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+send2trash==1.8.3
+sentencepiece==0.2.0
+sentry-sdk==1.44.0
+setproctitle==1.3.3
+setuptools==68.1.2
+shortuuid==1.0.13
+simple-websocket==1.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+stack-data==0.6.3
+starlette==0.37.2
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.8.0
+tensorflow-cpu==2.8.0
+tensorflow-io-gcs-filesystem==0.25.0
+tensorflow-io==0.25.0
+termcolor==2.4.0
+terminado==0.18.1
+terminaltables==3.1.10
+text-cutter==2.3.0
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.4.0
+thrift2pyi==1.0.3
+thriftpy2==0.4.20
+timm==0.9.16
+tinycss2==1.3.0
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+torch==2.1.2
+torchaudio==2.1.0+cu121
+torchlibrosa==0.1.0
+torchvision==0.16.0+cu121
+tornado==6.4
+tox==3.28.0
+tqdm==4.48.2
+traitlets==5.14.3
+transformers==4.39.2
+triton==2.1.0
+types-python-dateutil==2.9.0.20240316
+typing-extensions==4.10.0
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==1.26.13
+uvicorn==0.29.0
+uvloop==0.19.0
+virtualenv==20.25.1
+vllm==0.3.2
+watchdog==4.0.0
+watchfiles==0.21.0
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==12.0
+werkzeug==3.0.2
+wheel==0.41.2
+widgetsnbextension==4.0.10
+wrapt==1.16.0
+wsproto==1.2.0
+xformers==0.0.23.post1
+y-py==0.6.2
+yacs==0.1.8
+yapf==0.40.2
+ypy-websocket==0.8.4
+zipp==3.18.1
+zope.event==5.0
+zope.interface==6.2
+zstandard==0.19.0

None/verifier-reproduce/wandb/latest-run/files/wandb-metadata.json

@@ -0,0 +1,1026 @@
+{
+    "os": "Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31",
+    "python": "3.9.2",
+    "heartbeatAt": "2024-05-06T11:51:28.742440",
+    "startedAt": "2024-05-06T11:51:22.758194",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "../models/lean4_random_15k_all/checkpoint-236/",
+        "--data_dir",
+        "../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl",
+        "--data_id",
+        "lean4test",
+        "--target_set",
+        "train",
+        "--save_dir",
+        "../models/lean4_random_15k_all-sample10-osv-gt2",
+        "--generator_id",
+        "lean4_random_15k_all",
+        "--verifier_id",
+        "osv",
+        "--dedup",
+        "True",
+        "--per_problem_sampling_solution",
+        "10",
+        "--loss_level",
+        "token",
+        "--loss_on_llm",
+        "True",
+        "--num_train_epoches",
+        "1",
+        "--eval_steps",
+        "10000",
+        "--per_device_train_batch_size",
+        "16",
+        "--per_device_eval_batch_size",
+        "64",
+        "--gradient_accumulation_steps",
+        "4",
+        "--gradient_checkpointing",
+        "True",
+        "--learning_rate",
+        "2e-6",
+        "--weight_decay",
+        "0",
+        "--save_steps",
+        "1000",
+        "--lr_scheduler_type",
+        "linear",
+        "--warmup_steps",
+        "0",
+        "--save_epoches",
+        "1",
+        "--save_best",
+        "False",
+        "--save_total_limit",
+        "0",
+        "--logging_dir",
+        "None",
+        "--logging_steps",
+        "1",
+        "--easy",
+        "True",
+        "--seed",
+        "42"
+    ],
+    "state": "running",
+    "program": "/opt/tiger/auto-info/PSV/train_psv.py",
+    "codePath": "PSV/train_psv.py",
+    "host": "n122-200-048",
+    "username": "tiger",
+    "executable": "/usr/bin/python3",
+    "cpu_count": 90,
+    "cpu_count_logical": 180,
+    "cpu_freq": {
+        "current": 2599.5619999999913,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "total": 3519.748878479004,
+        "used": 517.5916595458984
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2932.9151611328125
+    }
+}

None/verifier-reproduce/wandb/latest-run/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss": 105.05744934082031, "v_loss": 104.5, "llm_loss": 0.5574514865875244, "lr": 1.989010989010989e-06, "_timestamp": 1714996346.1642962, "_runtime": 63.397871255874634, "_step": 3}

None/verifier-reproduce/wandb/latest-run/logs/debug-internal.log

@@ -0,0 +1,82 @@
+2024-05-06 19:51:22,768 INFO    StreamThr :1679810 [internal.py:wandb_internal():88] W&B internal server running at pid: 1679810, started at: 2024-05-06 19:51:22.767829
+2024-05-06 19:51:22,770 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: status
+2024-05-06 19:51:22,799 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: status
+2024-05-06 19:51:22,801 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: header
+2024-05-06 19:51:22,801 INFO    WriterThread:1679810 [datastore.py:open_for_write():77] open: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/run-gssu6m95.wandb
+2024-05-06 19:51:22,801 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: run
+2024-05-06 19:51:22,802 INFO    SenderThread:1679810 [tracking_sender.py:_maybe_setup_resume():588] checking resume status for None/verifier-reproduce/gssu6m95
+2024-05-06 19:51:23,641 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: check_version
+2024-05-06 19:51:23,644 INFO    SenderThread:1679810 [dir_watcher.py:__init__():216] watching files in: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files
+2024-05-06 19:51:23,644 INFO    SenderThread:1679810 [tracking_sender.py:_start_run_threads():934] run started: gssu6m95 with start time 1714996282.766425
+2024-05-06 19:51:23,644 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: summary
+2024-05-06 19:51:23,645 INFO    SenderThread:1679810 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-06 19:51:23,645 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: check_version
+2024-05-06 19:51:24,645 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/wandb-summary.json
+2024-05-06 19:51:28,645 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: run_start
+2024-05-06 19:51:28,686 DEBUG   HandlerThread:1679810 [system_info.py:__init__():31] System info init
+2024-05-06 19:51:28,686 DEBUG   HandlerThread:1679810 [system_info.py:__init__():46] System info init done
+2024-05-06 19:51:28,686 INFO    HandlerThread:1679810 [system_monitor.py:start():150] Starting system monitor
+2024-05-06 19:51:28,687 INFO    SystemMonitor:1679810 [system_monitor.py:_start():116] Starting system asset monitoring threads
+2024-05-06 19:51:28,687 INFO    HandlerThread:1679810 [system_monitor.py:probe():168] Collecting system info
+2024-05-06 19:51:28,687 INFO    SystemMonitor:1679810 [interfaces.py:start():168] Started cpu
+2024-05-06 19:51:28,687 INFO    SystemMonitor:1679810 [interfaces.py:start():168] Started disk
+2024-05-06 19:51:28,688 INFO    SystemMonitor:1679810 [interfaces.py:start():168] Started gpu
+2024-05-06 19:51:28,688 INFO    SystemMonitor:1679810 [interfaces.py:start():168] Started memory
+2024-05-06 19:51:28,689 INFO    SystemMonitor:1679810 [interfaces.py:start():168] Started network
+2024-05-06 19:51:28,742 DEBUG   HandlerThread:1679810 [system_info.py:probe():195] Probing system
+2024-05-06 19:51:28,748 DEBUG   HandlerThread:1679810 [git.py:repo():40] git repository is invalid
+2024-05-06 19:51:28,748 DEBUG   HandlerThread:1679810 [system_info.py:probe():241] Probing system done
+2024-05-06 19:51:28,748 DEBUG   HandlerThread:1679810 [system_monitor.py:probe():177] {'os': 'Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31', 'python': '3.9.2', 'heartbeatAt': '2024-05-06T11:51:28.742440', 'startedAt': '2024-05-06T11:51:22.758194', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path', '../models/lean4_random_15k_all/checkpoint-236/', '--data_dir', '../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl', '--data_id', 'lean4test', '--target_set', 'train', '--save_dir', '../models/lean4_random_15k_all-sample10-osv-gt2', '--generator_id', 'lean4_random_15k_all', '--verifier_id', 'osv', '--dedup', 'True', '--per_problem_sampling_solution', '10', '--loss_level', 'token', '--loss_on_llm', 'True', '--num_train_epoches', '1', '--eval_steps', '10000', '--per_device_train_batch_size', '16', '--per_device_eval_batch_size', '64', '--gradient_accumulation_steps', '4', '--gradient_checkpointing', 'True', '--learning_rate', '2e-6', '--weight_decay', '0', '--save_steps', '1000', '--lr_scheduler_type', 'linear', '--warmup_steps', '0', '--save_epoches', '1', '--save_best', 'False', '--save_total_limit', '0', '--logging_dir', 'None', '--logging_steps', '1', '--easy', 'True', '--seed', '42'), 'state': 'running', 'program': '/opt/tiger/auto-info/PSV/train_psv.py', 'codePath': 'PSV/train_psv.py', 'host': 'n122-200-048', 'username': 'tiger', 'executable': '/usr/bin/python3', 'cpu_count': 90, 'cpu_count_logical': 180, 'cpu_freq': {'current': 2599.5619999999913, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 3519.748878479004, 'used': 517.5916595458984}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2932.9151611328125}}
+2024-05-06 19:51:28,748 INFO    HandlerThread:1679810 [system_monitor.py:probe():178] Finished collecting system info
+2024-05-06 19:51:28,748 INFO    HandlerThread:1679810 [system_monitor.py:probe():181] Publishing system info
+2024-05-06 19:51:28,748 DEBUG   HandlerThread:1679810 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2024-05-06 19:51:28,749 DEBUG   HandlerThread:1679810 [system_info.py:_save_pip():67] Saving pip packages done
+2024-05-06 19:51:28,750 INFO    HandlerThread:1679810 [system_monitor.py:probe():183] Finished publishing system info
+2024-05-06 19:51:28,784 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: files
+2024-05-06 19:51:28,784 INFO    SenderThread:1679810 [tracking_sender.py:_save_file():1177] saving file wandb-metadata.json with policy now
+2024-05-06 19:51:28,788 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:51:28,788 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:51:28,789 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: telemetry
+2024-05-06 19:51:28,828 INFO    Thread-17 :1679810 [upload_job.py:push():143] Uploaded file /tmp/tmps749cj6lwandb/6c2e6y27-wandb-metadata.json
+2024-05-06 19:51:29,647 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/requirements.txt
+2024-05-06 19:51:29,647 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:51:29,647 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/wandb-metadata.json
+2024-05-06 19:51:31,648 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:51:43,791 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:51:43,791 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:51:47,651 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:51:54,653 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/config.yaml
+2024-05-06 19:51:58,793 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:51:58,794 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:52:07,440 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-06 19:52:09,657 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:52:13,795 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:52:13,796 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:52:16,219 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-06 19:52:16,220 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: history
+2024-05-06 19:52:16,220 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: summary
+2024-05-06 19:52:16,221 INFO    SenderThread:1679810 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-06 19:52:16,659 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/wandb-summary.json
+2024-05-06 19:52:17,660 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:52:26,205 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-06 19:52:26,206 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: history
+2024-05-06 19:52:26,206 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: summary
+2024-05-06 19:52:26,208 INFO    SenderThread:1679810 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-06 19:52:26,662 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/wandb-summary.json
+2024-05-06 19:52:27,662 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:52:28,690 DEBUG   SystemMonitor:1679810 [system_monitor.py:_start():130] Starting system metrics aggregation loop
+2024-05-06 19:52:28,693 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: stats
+2024-05-06 19:52:28,797 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:52:28,797 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:52:37,665 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log
+2024-05-06 19:52:43,799 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:52:43,799 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:52:58,415 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-06 19:52:58,416 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: history
+2024-05-06 19:52:58,416 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: summary
+2024-05-06 19:52:58,417 INFO    SenderThread:1679810 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-06 19:52:58,670 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/wandb-summary.json
+2024-05-06 19:52:58,695 DEBUG   SenderThread:1679810 [tracking_sender.py:send():315] send: stats
+2024-05-06 19:52:58,801 DEBUG   HandlerThread:1679810 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-06 19:52:58,801 DEBUG   SenderThread:1679810 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-06 19:52:59,671 INFO    Thread-15 :1679810 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/files/output.log

None/verifier-reproduce/wandb/latest-run/logs/debug.log

@@ -0,0 +1,24 @@
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_setup.py:_flush():68] Configure stats pid to 1674154
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/auto-info/wandb/settings
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'PSV/train_psv.py', 'program': '/opt/tiger/auto-info/PSV/train_psv.py'}
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/logs/debug.log
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240506_195122-gssu6m95/logs/debug-internal.log
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:init():518] calling init triggers
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:init():571] starting backend
+2024-05-06 19:51:22,760 INFO    MainThread:1674154 [wandb_init.py:init():575] setting up manager
+2024-05-06 19:51:22,764 INFO    MainThread:1674154 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-06 19:51:22,766 INFO    MainThread:1674154 [wandb_init.py:init():582] backend started and connected
+2024-05-06 19:51:22,769 INFO    MainThread:1674154 [wandb_init.py:init():660] updated telemetry
+2024-05-06 19:51:22,769 INFO    MainThread:1674154 [wandb_init.py:init():695] communicating run to backend with 60 second timeout
+2024-05-06 19:51:23,640 INFO    MainThread:1674154 [wandb_run.py:_on_init():1998] communicating current version
+2024-05-06 19:51:28,641 INFO    MainThread:1674154 [wandb_run.py:_on_init():2002] got version response None
+2024-05-06 19:51:28,641 INFO    MainThread:1674154 [wandb_init.py:init():730] starting run threads in backend
+2024-05-06 19:51:28,787 INFO    MainThread:1674154 [wandb_run.py:_console_start():1979] atexit reg
+2024-05-06 19:51:28,787 INFO    MainThread:1674154 [wandb_run.py:_redirect():1837] redirect: SettingsConsole.WRAP_RAW
+2024-05-06 19:51:28,787 INFO    MainThread:1674154 [wandb_run.py:_redirect():1902] Wrapping output streams.
+2024-05-06 19:51:28,787 INFO    MainThread:1674154 [wandb_run.py:_redirect():1924] Redirects installed.
+2024-05-06 19:51:28,788 INFO    MainThread:1674154 [wandb_init.py:init():767] run started, returning control to user process

None/verifier-reproduce/wandb/latest-run/tracking_info.json

@@ -0,0 +1 @@
+{"project_id": "", "run_id": "gssu6m95", "run_link": "https://ml.bytedance.net/experiment/tracking/detail?Id=project_20240412_f2edc88c&selectedTrial=gssu6m95"}

None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/files/config.yaml

@@ -0,0 +1,27 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.13.72
+    framework: huggingface
+    huggingface_version: 4.39.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.9.2
+    start_time: 1714745564.532508
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 14
+      - 23
+      4: 3.9.2
+      5: 0.13.72
+      6: 4.39.2
+      8:
+      - 5

None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/logs/debug-internal.log

@@ -0,0 +1,12 @@
+2024-05-03 22:12:44,535 INFO    StreamThr :355131 [internal.py:wandb_internal():88] W&B internal server running at pid: 355131, started at: 2024-05-03 22:12:44.535085
+2024-05-03 22:12:44,537 DEBUG   HandlerThread:355131 [handler.py:handle_request():140] handle_request: status
+2024-05-03 22:12:44,566 DEBUG   SenderThread:355131 [tracking_sender.py:send_request():329] send_request: status
+2024-05-03 22:12:44,854 INFO    WriterThread:355131 [datastore.py:open_for_write():77] open: None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/run-5gmak6l6.wandb
+2024-05-03 22:12:44,854 DEBUG   SenderThread:355131 [tracking_sender.py:send():315] send: header
+2024-05-03 22:12:44,854 DEBUG   SenderThread:355131 [tracking_sender.py:send():315] send: run
+2024-05-03 22:12:44,857 INFO    SenderThread:355131 [tracking_sender.py:_maybe_setup_resume():588] checking resume status for None/verifier-reproduce/5gmak6l6
+2024-05-03 22:12:45,655 INFO    SenderThread:355131 [dir_watcher.py:__init__():216] watching files in: None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/files
+2024-05-03 22:12:45,655 INFO    SenderThread:355131 [tracking_sender.py:_start_run_threads():934] run started: 5gmak6l6 with start time 1714745564.532508
+2024-05-03 22:12:51,854 INFO    MainThread:355131 [internal.py:handle_exit():78] Internal process exited
+2024-05-03 22:12:51,855 WARNING MsgRouterThr:355131 [router.py:message_loop():74] EOFError seen in message_loop
+2024-05-03 22:12:51,855 INFO    WriterThread:355131 [datastore.py:close():283] close: None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/run-5gmak6l6.wandb

None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/logs/debug.log

@@ -0,0 +1,16 @@
+2024-05-03 22:12:44,527 INFO    MainThread:345011 [wandb_setup.py:_flush():68] Configure stats pid to 345011
+2024-05-03 22:12:44,527 INFO    MainThread:345011 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-03 22:12:44,527 INFO    MainThread:345011 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/auto-info/wandb/settings
+2024-05-03 22:12:44,527 INFO    MainThread:345011 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-03 22:12:44,527 INFO    MainThread:345011 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'PSV/train_psv.py', 'program': '/opt/tiger/auto-info/PSV/train_psv.py'}
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/logs/debug.log
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/logs/debug-internal.log
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:init():518] calling init triggers
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:init():571] starting backend
+2024-05-03 22:12:44,528 INFO    MainThread:345011 [wandb_init.py:init():575] setting up manager
+2024-05-03 22:12:44,530 INFO    MainThread:345011 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-03 22:12:44,532 INFO    MainThread:345011 [wandb_init.py:init():582] backend started and connected
+2024-05-03 22:12:44,535 INFO    MainThread:345011 [wandb_init.py:init():660] updated telemetry
+2024-05-03 22:12:44,536 INFO    MainThread:345011 [wandb_init.py:init():695] communicating run to backend with 60 second timeout

None/verifier-reproduce/wandb/run-20240503_221244-5gmak6l6/tracking_info.json

@@ -0,0 +1 @@
+{"project_id": "", "run_id": "5gmak6l6", "run_link": "https://ml.bytedance.net/experiment/tracking/detail?Id=project_20240412_f2edc88c&selectedTrial=5gmak6l6"}

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/config.yaml

@@ -0,0 +1,27 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.13.72
+    framework: huggingface
+    huggingface_version: 4.39.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.9.2
+    start_time: 1714745787.936242
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 14
+      - 23
+      4: 3.9.2
+      5: 0.13.72
+      6: 4.39.2
+      8:
+      - 5

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/output.log

@@ -0,0 +1 @@
+

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/requirements.txt

@@ -0,0 +1,346 @@
+absl-py==0.15.0
+accelerate==0.29.3
+addict==2.4.0
+aiofiles==22.1.0
+aioprometheus==23.12.0
+aiosignal==1.3.1
+aiosqlite==0.20.0
+annotated-types==0.6.0
+anyio==4.3.0
+apex==0.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+babel==2.14.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+byted-dataloader==0.4.4
+byted-encrypted-hdfs==0.5.5
+byted-iceberg==0.2.112
+byted-lafka-internal==1.4.16rc1
+byted-remote-ikernel==0.4.8
+byted-torch-monitor==0.0.1
+byted-torch==2.1.0.post2
+byted-wandb==0.13.72
+bytedance-context==0.7.1
+bytedance-metrics==0.5.1
+bytedance.modelhub==0.0.63
+bytedance.servicediscovery==0.1.2
+bytedbackgrounds==0.0.6
+byteddatabus==1.0.6
+byteddps==0.1.2
+bytedenv==0.6.2
+bytedeuler==0.42.2
+bytedfeather==0.3.5
+bytedkafka==0.2.9
+bytedkms==3.2.9
+bytedkmsv2==0.9.35
+bytedlego-ops==1.0.0.218
+bytedlogger==0.15.1
+bytedlogid==0.2.1
+bytedmemfd==0.2
+bytedmetrics==0.9.1
+bytedmoxing==0.2.0
+bytedpymongo==2.0.5
+bytedrh2==1.18.9
+bytedservicediscovery==0.17.4
+bytedsinfmetacenter==1.4.3
+bytedtcc==1.4.2
+bytedtitan==1.0.0
+bytedtos==1.1.16
+bytedtrace==0.3.0
+bytedztijwthelper==0.0.22
+bytedztispiffe==0.0.11
+cachetools==5.3.3
+certifi==2022.12.7
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.1.1
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.0
+cramjam==2.8.3
+crcmod==1.7
+cryptography==38.0.4
+cupy-cuda12x==12.1.0
+cycler==0.12.1
+dbus-python==1.2.16
+debugpy==1.8.1
+decorator==5.1.1
+deepspeed==0.14.0
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.8
+distlib==0.3.8
+distro-info==1.0
+dm-tree==0.1.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring-parser==0.16
+einops==0.7.0
+entrypoints==0.4
+exceptiongroup==1.2.1
+executing==2.0.1
+fastapi==0.110.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.3
+flash-attn==2.5.6
+flatbuffers==24.3.25
+fonttools==4.50.0
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+fvcore==0.1.5.post20221221
+gast==0.5.4
+gevent==22.10.2
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.29.0
+google-pasta==0.2.0
+greenlet==3.0.3
+grpcio==1.62.1
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+httptools==0.6.1
+huggingface-hub==0.22.2
+idna==3.4
+importlib-metadata==7.1.0
+importlib-resources==6.4.0
+iniconfig==2.0.0
+iopath==0.1.10
+iotop==0.6
+ipaddress==1.0.23
+ipykernel==6.29.4
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.2
+isoduration==20.11.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.25
+jsonargparse==4.14.0
+jsonpath-ng==1.5.3
+jsonpointer==2.4
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-client==7.0.0
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyter-events==0.10.0
+jupyter-server-fileid==0.9.2
+jupyter-server-terminals==0.5.3
+jupyter-server-ydoc==0.8.0
+jupyter-server==2.14.0
+jupyter-ydoc==0.2.5
+jupyter==1.0.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.27.1
+jupyterlab-widgets==3.0.10
+jupyterlab==3.6.4
+keras-preprocessing==1.1.2
+keras==2.8.0
+kiwisolver==1.4.5
+lazy-loader==0.3
+libclang==18.1.1
+librosa==0.10.1
+llvmlite==0.42.0
+markdown-it-py==3.0.0
+markdown==3.6
+markupsafe==2.1.2
+matplotlib-inline==0.1.7
+matplotlib==3.8.3
+mdurl==0.1.2
+mistune==3.0.2
+mmhash3==3.0.1
+mpmath==1.2.1
+msgpack==1.0.8
+nbclassic==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+notebook-shim==0.2.4
+notebook==6.5.7
+numba==0.59.1
+numpy==1.25.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-dali-cuda120==1.36.0
+nvidia-nccl-cu12==2.18.1
+nvidia-nvimgcodec-cu12==0.2.0.7
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+onnx==1.16.0
+opt-einsum==3.3.0
+orjson==3.10.2
+overrides==7.7.0
+packaging==24.0
+pandas==1.3.5
+pandocfilters==1.5.1
+parso==0.8.4
+pathlib2==2.3.7.post1
+pathtools==0.1.2
+pexpect==4.8.0
+pillow==9.3.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+pooch==1.8.1
+portalocker==2.8.2
+prettytable==3.10.0
+prometheus-client==0.20.0
+promise==2.3
+prompt-toolkit==3.0.43
+protobuf==3.20.0
+psutil==5.9.8
+ptflops==0.7.2.2
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py==1.11.0
+pyarrow==11.0.0
+pyasn1-modules==0.4.0
+pyasn1==0.6.0
+pybind11==2.12.0
+pycparser==2.22
+pycryptodomex==3.20.0
+pycurl==7.43.0.6
+pydantic-core==2.18.2
+pydantic==2.7.1
+pygments==2.17.2
+pygobject==3.38.0
+pyjwt==2.8.0
+pynvml==11.5.0
+pyopenssl==22.1.0
+pyparsing==3.1.2
+pypeg2==2.15.2
+pytest==6.2.5
+python-apt==2.2.1
+python-consul==1.1.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-engineio==4.9.0
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-snappy==0.7.1
+python-socketio==5.11.2
+pytz==2024.1
+pyyaml==5.4.1
+pyzmq==26.0.3
+qtconsole==5.5.1
+qtpy==2.4.1
+quantile-python==1.1
+ray==2.20.0
+redis==4.5.5
+referencing==0.35.1
+regex==2023.12.25
+requests-oauthlib==2.0.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.0
+rsa==4.9
+safetensors==0.4.2
+schedule==1.2.1
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+send2trash==1.8.3
+sentencepiece==0.2.0
+sentry-sdk==1.44.0
+setproctitle==1.3.3
+setuptools==68.1.2
+shortuuid==1.0.13
+simple-websocket==1.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+stack-data==0.6.3
+starlette==0.37.2
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.8.0
+tensorflow-cpu==2.8.0
+tensorflow-io-gcs-filesystem==0.25.0
+tensorflow-io==0.25.0
+termcolor==2.4.0
+terminado==0.18.1
+terminaltables==3.1.10
+text-cutter==2.3.0
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.4.0
+thrift2pyi==1.0.3
+thriftpy2==0.4.20
+timm==0.9.16
+tinycss2==1.3.0
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+torch==2.1.2
+torchaudio==2.1.0+cu121
+torchlibrosa==0.1.0
+torchvision==0.16.0+cu121
+tornado==6.4
+tox==3.28.0
+tqdm==4.48.2
+traitlets==5.14.3
+transformers==4.39.2
+triton==2.1.0
+types-python-dateutil==2.9.0.20240316
+typing-extensions==4.10.0
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==1.26.13
+uvicorn==0.29.0
+uvloop==0.19.0
+virtualenv==20.25.1
+vllm==0.3.2
+watchdog==4.0.0
+watchfiles==0.21.0
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==12.0
+werkzeug==3.0.2
+wheel==0.41.2
+widgetsnbextension==4.0.10
+wrapt==1.16.0
+wsproto==1.2.0
+xformers==0.0.23.post1
+y-py==0.6.2
+yacs==0.1.8
+yapf==0.40.2
+ypy-websocket==0.8.4
+zipp==3.18.1
+zope.event==5.0
+zope.interface==6.2
+zstandard==0.19.0

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/wandb-metadata.json

@@ -0,0 +1,1026 @@
+{
+    "os": "Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31",
+    "python": "3.9.2",
+    "heartbeatAt": "2024-05-03T14:16:34.053489",
+    "startedAt": "2024-05-03T14:16:27.931131",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "../models/lean4_random_15k_all/checkpoint-236/",
+        "--data_dir",
+        "../repl/pass_rate_results/lean4_basic_test/lean4_random_15k_allpass5.jsonl",
+        "--data_id",
+        "lean4test",
+        "--target_set",
+        "train",
+        "--save_dir",
+        "../models/lean4_random_15k_all-sample5-test-gt",
+        "--generator_id",
+        "lean4_random_15k_all",
+        "--verifier_id",
+        "test",
+        "--dedup",
+        "True",
+        "--per_problem_sampling_solution",
+        "5",
+        "--loss_level",
+        "token",
+        "--loss_on_llm",
+        "True",
+        "--num_train_epoches",
+        "1",
+        "--eval_steps",
+        "10000",
+        "--per_device_train_batch_size",
+        "16",
+        "--per_device_eval_batch_size",
+        "64",
+        "--gradient_accumulation_steps",
+        "4",
+        "--gradient_checkpointing",
+        "True",
+        "--learning_rate",
+        "2e-6",
+        "--weight_decay",
+        "0",
+        "--save_steps",
+        "1000",
+        "--lr_scheduler_type",
+        "linear",
+        "--warmup_steps",
+        "0",
+        "--save_epoches",
+        "1",
+        "--save_best",
+        "False",
+        "--save_total_limit",
+        "0",
+        "--logging_dir",
+        "None",
+        "--logging_steps",
+        "1",
+        "--easy",
+        "True",
+        "--seed",
+        "42"
+    ],
+    "state": "running",
+    "program": "/opt/tiger/auto-info/PSV/train_psv.py",
+    "codePath": "PSV/train_psv.py",
+    "host": "n122-200-048",
+    "username": "tiger",
+    "executable": "/usr/bin/python3",
+    "cpu_count": 90,
+    "cpu_count_logical": 180,
+    "cpu_freq": {
+        "current": 2599.5619999999913,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "total": 3519.748878479004,
+        "used": 530.565601348877
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2932.9151611328125
+    }
+}

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/logs/debug-internal.log

@@ -0,0 +1,46 @@
+2024-05-03 22:16:27,939 INFO    StreamThr :402492 [internal.py:wandb_internal():88] W&B internal server running at pid: 402492, started at: 2024-05-03 22:16:27.938574
+2024-05-03 22:16:27,940 DEBUG   HandlerThread:402492 [handler.py:handle_request():140] handle_request: status
+2024-05-03 22:16:27,965 DEBUG   SenderThread:402492 [tracking_sender.py:send_request():329] send_request: status
+2024-05-03 22:16:27,967 DEBUG   SenderThread:402492 [tracking_sender.py:send():315] send: header
+2024-05-03 22:16:27,967 INFO    WriterThread:402492 [datastore.py:open_for_write():77] open: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/run-nsl0hmdh.wandb
+2024-05-03 22:16:27,967 DEBUG   SenderThread:402492 [tracking_sender.py:send():315] send: run
+2024-05-03 22:16:27,969 INFO    SenderThread:402492 [tracking_sender.py:_maybe_setup_resume():588] checking resume status for None/verifier-reproduce/nsl0hmdh
+2024-05-03 22:16:28,948 DEBUG   HandlerThread:402492 [handler.py:handle_request():140] handle_request: check_version
+2024-05-03 22:16:28,951 INFO    SenderThread:402492 [dir_watcher.py:__init__():216] watching files in: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files
+2024-05-03 22:16:28,951 INFO    SenderThread:402492 [tracking_sender.py:_start_run_threads():934] run started: nsl0hmdh with start time 1714745787.936242
+2024-05-03 22:16:28,951 DEBUG   SenderThread:402492 [tracking_sender.py:send():315] send: summary
+2024-05-03 22:16:28,951 INFO    SenderThread:402492 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-03 22:16:28,951 DEBUG   SenderThread:402492 [tracking_sender.py:send_request():329] send_request: check_version
+2024-05-03 22:16:29,952 INFO    Thread-15 :402492 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/wandb-summary.json
+2024-05-03 22:16:33,951 DEBUG   HandlerThread:402492 [handler.py:handle_request():140] handle_request: run_start
+2024-05-03 22:16:33,996 DEBUG   HandlerThread:402492 [system_info.py:__init__():31] System info init
+2024-05-03 22:16:33,996 DEBUG   HandlerThread:402492 [system_info.py:__init__():46] System info init done
+2024-05-03 22:16:33,996 INFO    HandlerThread:402492 [system_monitor.py:start():150] Starting system monitor
+2024-05-03 22:16:33,997 INFO    SystemMonitor:402492 [system_monitor.py:_start():116] Starting system asset monitoring threads
+2024-05-03 22:16:33,997 INFO    HandlerThread:402492 [system_monitor.py:probe():168] Collecting system info
+2024-05-03 22:16:33,997 INFO    SystemMonitor:402492 [interfaces.py:start():168] Started cpu
+2024-05-03 22:16:33,998 INFO    SystemMonitor:402492 [interfaces.py:start():168] Started disk
+2024-05-03 22:16:33,999 INFO    SystemMonitor:402492 [interfaces.py:start():168] Started gpu
+2024-05-03 22:16:33,999 INFO    SystemMonitor:402492 [interfaces.py:start():168] Started memory
+2024-05-03 22:16:33,999 INFO    SystemMonitor:402492 [interfaces.py:start():168] Started network
+2024-05-03 22:16:34,053 DEBUG   HandlerThread:402492 [system_info.py:probe():195] Probing system
+2024-05-03 22:16:34,061 DEBUG   HandlerThread:402492 [git.py:repo():40] git repository is invalid
+2024-05-03 22:16:34,061 DEBUG   HandlerThread:402492 [system_info.py:probe():241] Probing system done
+2024-05-03 22:16:34,061 DEBUG   HandlerThread:402492 [system_monitor.py:probe():177] {'os': 'Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31', 'python': '3.9.2', 'heartbeatAt': '2024-05-03T14:16:34.053489', 'startedAt': '2024-05-03T14:16:27.931131', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path', '../models/lean4_random_15k_all/checkpoint-236/', '--data_dir', '../repl/pass_rate_results/lean4_basic_test/lean4_random_15k_allpass5.jsonl', '--data_id', 'lean4test', '--target_set', 'train', '--save_dir', '../models/lean4_random_15k_all-sample5-test-gt', '--generator_id', 'lean4_random_15k_all', '--verifier_id', 'test', '--dedup', 'True', '--per_problem_sampling_solution', '5', '--loss_level', 'token', '--loss_on_llm', 'True', '--num_train_epoches', '1', '--eval_steps', '10000', '--per_device_train_batch_size', '16', '--per_device_eval_batch_size', '64', '--gradient_accumulation_steps', '4', '--gradient_checkpointing', 'True', '--learning_rate', '2e-6', '--weight_decay', '0', '--save_steps', '1000', '--lr_scheduler_type', 'linear', '--warmup_steps', '0', '--save_epoches', '1', '--save_best', 'False', '--save_total_limit', '0', '--logging_dir', 'None', '--logging_steps', '1', '--easy', 'True', '--seed', '42'), 'state': 'running', 'program': '/opt/tiger/auto-info/PSV/train_psv.py', 'codePath': 'PSV/train_psv.py', 'host': 'n122-200-048', 'username': 'tiger', 'executable': '/usr/bin/python3', 'cpu_count': 90, 'cpu_count_logical': 180, 'cpu_freq': {'current': 2599.5619999999913, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 3519.748878479004, 'used': 530.565601348877}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2932.9151611328125}}
+2024-05-03 22:16:34,062 INFO    HandlerThread:402492 [system_monitor.py:probe():178] Finished collecting system info
+2024-05-03 22:16:34,062 INFO    HandlerThread:402492 [system_monitor.py:probe():181] Publishing system info
+2024-05-03 22:16:34,062 DEBUG   HandlerThread:402492 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2024-05-03 22:16:34,062 DEBUG   HandlerThread:402492 [system_info.py:_save_pip():67] Saving pip packages done
+2024-05-03 22:16:34,064 INFO    HandlerThread:402492 [system_monitor.py:probe():183] Finished publishing system info
+2024-05-03 22:16:34,101 DEBUG   SenderThread:402492 [tracking_sender.py:send():315] send: files
+2024-05-03 22:16:34,102 INFO    SenderThread:402492 [tracking_sender.py:_save_file():1177] saving file wandb-metadata.json with policy now
+2024-05-03 22:16:34,105 DEBUG   HandlerThread:402492 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-03 22:16:34,106 DEBUG   SenderThread:402492 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-03 22:16:34,106 DEBUG   SenderThread:402492 [tracking_sender.py:send():315] send: telemetry
+2024-05-03 22:16:34,229 INFO    Thread-17 :402492 [upload_job.py:push():143] Uploaded file /tmp/tmpl7loydbnwandb/xuedt8ej-wandb-metadata.json
+2024-05-03 22:16:34,952 INFO    Thread-15 :402492 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/requirements.txt
+2024-05-03 22:16:34,953 INFO    Thread-15 :402492 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/output.log
+2024-05-03 22:16:34,953 INFO    Thread-15 :402492 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/wandb-metadata.json
+2024-05-03 22:16:36,953 INFO    Thread-15 :402492 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/files/output.log
+2024-05-03 22:16:47,953 INFO    MainThread:402492 [internal.py:handle_exit():78] Internal process exited
+2024-05-03 22:16:47,953 WARNING MsgRouterThr:402492 [router.py:message_loop():74] EOFError seen in message_loop

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/logs/debug.log

@@ -0,0 +1,24 @@
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_setup.py:_flush():68] Configure stats pid to 391503
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_setup.py:_flush():68] Loading settings from /home/tiger/.config/wandb/settings
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_setup.py:_flush():68] Loading settings from /opt/tiger/auto-info/wandb/settings
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_setup.py:_flush():68] Loading settings from environment variables: {'_require_service': 'True', 'project': 'verifier-reproduce'}
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_setup.py:_flush():68] Inferring run settings from compute environment: {'program_relpath': 'PSV/train_psv.py', 'program': '/opt/tiger/auto-info/PSV/train_psv.py'}
+2024-05-03 22:16:27,932 INFO    MainThread:391503 [wandb_init.py:_log_setup():478] Logging user logs to None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/logs/debug.log
+2024-05-03 22:16:27,933 INFO    MainThread:391503 [wandb_init.py:_log_setup():479] Logging internal logs to None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/logs/debug-internal.log
+2024-05-03 22:16:27,933 INFO    MainThread:391503 [wandb_init.py:init():518] calling init triggers
+2024-05-03 22:16:27,933 INFO    MainThread:391503 [wandb_init.py:init():521] wandb.init called with sweep_config: {}
+config: {}
+2024-05-03 22:16:27,933 INFO    MainThread:391503 [wandb_init.py:init():571] starting backend
+2024-05-03 22:16:27,933 INFO    MainThread:391503 [wandb_init.py:init():575] setting up manager
+2024-05-03 22:16:27,934 INFO    MainThread:391503 [backend.py:_multiprocessing_setup():102] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-05-03 22:16:27,936 INFO    MainThread:391503 [wandb_init.py:init():582] backend started and connected
+2024-05-03 22:16:27,938 INFO    MainThread:391503 [wandb_init.py:init():660] updated telemetry
+2024-05-03 22:16:27,939 INFO    MainThread:391503 [wandb_init.py:init():695] communicating run to backend with 60 second timeout
+2024-05-03 22:16:28,947 INFO    MainThread:391503 [wandb_run.py:_on_init():1998] communicating current version
+2024-05-03 22:16:33,948 INFO    MainThread:391503 [wandb_run.py:_on_init():2002] got version response None
+2024-05-03 22:16:33,948 INFO    MainThread:391503 [wandb_init.py:init():730] starting run threads in backend
+2024-05-03 22:16:34,104 INFO    MainThread:391503 [wandb_run.py:_console_start():1979] atexit reg
+2024-05-03 22:16:34,104 INFO    MainThread:391503 [wandb_run.py:_redirect():1837] redirect: SettingsConsole.WRAP_RAW
+2024-05-03 22:16:34,104 INFO    MainThread:391503 [wandb_run.py:_redirect():1902] Wrapping output streams.
+2024-05-03 22:16:34,104 INFO    MainThread:391503 [wandb_run.py:_redirect():1924] Redirects installed.
+2024-05-03 22:16:34,104 INFO    MainThread:391503 [wandb_init.py:init():767] run started, returning control to user process

None/verifier-reproduce/wandb/run-20240503_221627-nsl0hmdh/tracking_info.json

@@ -0,0 +1 @@
+{"project_id": "", "run_id": "nsl0hmdh", "run_link": "https://ml.bytedance.net/experiment/tracking/detail?Id=project_20240412_f2edc88c&selectedTrial=nsl0hmdh"}

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/config.yaml

@@ -0,0 +1 @@
+wandb_version: 1

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log

@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/requirements.txt

@@ -0,0 +1,346 @@
+absl-py==0.15.0
+accelerate==0.29.3
+addict==2.4.0
+aiofiles==22.1.0
+aioprometheus==23.12.0
+aiosignal==1.3.1
+aiosqlite==0.20.0
+annotated-types==0.6.0
+anyio==4.3.0
+apex==0.1
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+babel==2.14.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+byted-dataloader==0.4.4
+byted-encrypted-hdfs==0.5.5
+byted-iceberg==0.2.112
+byted-lafka-internal==1.4.16rc1
+byted-remote-ikernel==0.4.8
+byted-torch-monitor==0.0.1
+byted-torch==2.1.0.post2
+byted-wandb==0.13.72
+bytedance-context==0.7.1
+bytedance-metrics==0.5.1
+bytedance.modelhub==0.0.63
+bytedance.servicediscovery==0.1.2
+bytedbackgrounds==0.0.6
+byteddatabus==1.0.6
+byteddps==0.1.2
+bytedenv==0.6.2
+bytedeuler==0.42.2
+bytedfeather==0.3.5
+bytedkafka==0.2.9
+bytedkms==3.2.9
+bytedkmsv2==0.9.35
+bytedlego-ops==1.0.0.218
+bytedlogger==0.15.1
+bytedlogid==0.2.1
+bytedmemfd==0.2
+bytedmetrics==0.9.1
+bytedmoxing==0.2.0
+bytedpymongo==2.0.5
+bytedrh2==1.18.9
+bytedservicediscovery==0.17.4
+bytedsinfmetacenter==1.4.3
+bytedtcc==1.4.2
+bytedtitan==1.0.0
+bytedtos==1.1.16
+bytedtrace==0.3.0
+bytedztijwthelper==0.0.22
+bytedztispiffe==0.0.11
+cachetools==5.3.3
+certifi==2022.12.7
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.1.1
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.0
+cramjam==2.8.3
+crcmod==1.7
+cryptography==38.0.4
+cupy-cuda12x==12.1.0
+cycler==0.12.1
+dbus-python==1.2.16
+debugpy==1.8.1
+decorator==5.1.1
+deepspeed==0.14.0
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.8
+distlib==0.3.8
+distro-info==1.0
+dm-tree==0.1.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+docstring-parser==0.16
+einops==0.7.0
+entrypoints==0.4
+exceptiongroup==1.2.1
+executing==2.0.1
+fastapi==0.110.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.3
+flash-attn==2.5.6
+flatbuffers==24.3.25
+fonttools==4.50.0
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+fvcore==0.1.5.post20221221
+gast==0.5.4
+gevent==22.10.2
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.29.0
+google-pasta==0.2.0
+greenlet==3.0.3
+grpcio==1.62.1
+gunicorn==20.1.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+httptools==0.6.1
+huggingface-hub==0.22.2
+idna==3.4
+importlib-metadata==7.1.0
+importlib-resources==6.4.0
+iniconfig==2.0.0
+iopath==0.1.10
+iotop==0.6
+ipaddress==1.0.23
+ipykernel==6.29.4
+ipython-genutils==0.2.0
+ipython==8.18.1
+ipywidgets==8.1.2
+isoduration==20.11.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.25
+jsonargparse==4.14.0
+jsonpath-ng==1.5.3
+jsonpointer==2.4
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-client==7.0.0
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyter-events==0.10.0
+jupyter-server-fileid==0.9.2
+jupyter-server-terminals==0.5.3
+jupyter-server-ydoc==0.8.0
+jupyter-server==2.14.0
+jupyter-ydoc==0.2.5
+jupyter==1.0.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.27.1
+jupyterlab-widgets==3.0.10
+jupyterlab==3.6.4
+keras-preprocessing==1.1.2
+keras==2.8.0
+kiwisolver==1.4.5
+lazy-loader==0.3
+libclang==18.1.1
+librosa==0.10.1
+llvmlite==0.42.0
+markdown-it-py==3.0.0
+markdown==3.6
+markupsafe==2.1.2
+matplotlib-inline==0.1.7
+matplotlib==3.8.3
+mdurl==0.1.2
+mistune==3.0.2
+mmhash3==3.0.1
+mpmath==1.2.1
+msgpack==1.0.8
+nbclassic==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+notebook-shim==0.2.4
+notebook==6.5.7
+numba==0.59.1
+numpy==1.25.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-dali-cuda120==1.36.0
+nvidia-nccl-cu12==2.18.1
+nvidia-nvimgcodec-cu12==0.2.0.7
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+onnx==1.16.0
+opt-einsum==3.3.0
+orjson==3.10.2
+overrides==7.7.0
+packaging==24.0
+pandas==1.3.5
+pandocfilters==1.5.1
+parso==0.8.4
+pathlib2==2.3.7.post1
+pathtools==0.1.2
+pexpect==4.8.0
+pillow==9.3.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+pooch==1.8.1
+portalocker==2.8.2
+prettytable==3.10.0
+prometheus-client==0.20.0
+promise==2.3
+prompt-toolkit==3.0.43
+protobuf==3.20.0
+psutil==5.9.8
+ptflops==0.7.2.2
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py==1.11.0
+pyarrow==11.0.0
+pyasn1-modules==0.4.0
+pyasn1==0.6.0
+pybind11==2.12.0
+pycparser==2.22
+pycryptodomex==3.20.0
+pycurl==7.43.0.6
+pydantic-core==2.18.2
+pydantic==2.7.1
+pygments==2.17.2
+pygobject==3.38.0
+pyjwt==2.8.0
+pynvml==11.5.0
+pyopenssl==22.1.0
+pyparsing==3.1.2
+pypeg2==2.15.2
+pytest==6.2.5
+python-apt==2.2.1
+python-consul==1.1.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-engineio==4.9.0
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-snappy==0.7.1
+python-socketio==5.11.2
+pytz==2024.1
+pyyaml==5.4.1
+pyzmq==26.0.3
+qtconsole==5.5.1
+qtpy==2.4.1
+quantile-python==1.1
+ray==2.20.0
+redis==4.5.5
+referencing==0.35.1
+regex==2023.12.25
+requests-oauthlib==2.0.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.0
+rsa==4.9
+safetensors==0.4.2
+schedule==1.2.1
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+send2trash==1.8.3
+sentencepiece==0.2.0
+sentry-sdk==1.44.0
+setproctitle==1.3.3
+setuptools==68.1.2
+shortuuid==1.0.13
+simple-websocket==1.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+stack-data==0.6.3
+starlette==0.37.2
+sympy==1.11.1
+tabulate==0.9.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.8.0
+tensorflow-cpu==2.8.0
+tensorflow-io-gcs-filesystem==0.25.0
+tensorflow-io==0.25.0
+termcolor==2.4.0
+terminado==0.18.1
+terminaltables==3.1.10
+text-cutter==2.3.0
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.4.0
+thrift2pyi==1.0.3
+thriftpy2==0.4.20
+timm==0.9.16
+tinycss2==1.3.0
+tokenizers==0.15.2
+toml==0.10.2
+tomli==2.0.1
+torch==2.1.2
+torchaudio==2.1.0+cu121
+torchlibrosa==0.1.0
+torchvision==0.16.0+cu121
+tornado==6.4
+tox==3.28.0
+tqdm==4.48.2
+traitlets==5.14.3
+transformers==4.39.2
+triton==2.1.0
+types-python-dateutil==2.9.0.20240316
+typing-extensions==4.10.0
+unattended-upgrades==0.1
+uri-template==1.3.0
+urllib3==1.26.13
+uvicorn==0.29.0
+uvloop==0.19.0
+virtualenv==20.25.1
+vllm==0.3.2
+watchdog==4.0.0
+watchfiles==0.21.0
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==12.0
+werkzeug==3.0.2
+wheel==0.41.2
+widgetsnbextension==4.0.10
+wrapt==1.16.0
+wsproto==1.2.0
+xformers==0.0.23.post1
+y-py==0.6.2
+yacs==0.1.8
+yapf==0.40.2
+ypy-websocket==0.8.4
+zipp==3.18.1
+zope.event==5.0
+zope.interface==6.2
+zstandard==0.19.0

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-metadata.json

@@ -0,0 +1,1026 @@
+{
+    "os": "Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31",
+    "python": "3.9.2",
+    "heartbeatAt": "2024-05-04T02:14:43.442670",
+    "startedAt": "2024-05-04T02:14:37.405615",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_name_or_path",
+        "../models/lean4_random_15k_all/checkpoint-236/",
+        "--data_dir",
+        "../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl",
+        "--data_id",
+        "lean4test",
+        "--target_set",
+        "train",
+        "--save_dir",
+        "../models/lean4_random_15k_all-sample10-osv-gt",
+        "--generator_id",
+        "lean4_random_15k_all",
+        "--verifier_id",
+        "osv",
+        "--dedup",
+        "True",
+        "--per_problem_sampling_solution",
+        "10",
+        "--loss_level",
+        "token",
+        "--loss_on_llm",
+        "True",
+        "--num_train_epoches",
+        "1",
+        "--eval_steps",
+        "10000",
+        "--per_device_train_batch_size",
+        "16",
+        "--per_device_eval_batch_size",
+        "64",
+        "--gradient_accumulation_steps",
+        "4",
+        "--gradient_checkpointing",
+        "True",
+        "--learning_rate",
+        "2e-6",
+        "--weight_decay",
+        "0",
+        "--save_steps",
+        "1000",
+        "--lr_scheduler_type",
+        "linear",
+        "--warmup_steps",
+        "0",
+        "--save_epoches",
+        "1",
+        "--save_best",
+        "False",
+        "--save_total_limit",
+        "0",
+        "--logging_dir",
+        "None",
+        "--logging_steps",
+        "1",
+        "--easy",
+        "True",
+        "--seed",
+        "42"
+    ],
+    "state": "running",
+    "program": "/opt/tiger/auto-info/PSV/train_psv.py",
+    "codePath": "PSV/train_psv.py",
+    "host": "n122-200-048",
+    "username": "tiger",
+    "executable": "/usr/bin/python3",
+    "cpu_count": 90,
+    "cpu_count_logical": 180,
+    "cpu_freq": {
+        "current": 2599.5619999999913,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.562,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "total": 3519.748878479004,
+        "used": 492.3821258544922
+    },
+    "gpu": "NVIDIA H800",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H800",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 2932.9151611328125
+    }
+}

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss": 52.92954635620117, "v_loss": 52.25, "llm_loss": 0.6795449256896973, "lr": 1.978021978021978e-06, "_timestamp": 1714789022.2754493, "_runtime": 144.86294221878052, "_step": 9}

None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/logs/debug-internal.log

@@ -0,0 +1,124 @@
+2024-05-04 10:14:37,414 INFO    StreamThr :3810790 [internal.py:wandb_internal():88] W&B internal server running at pid: 3810790, started at: 2024-05-04 10:14:37.414245
+2024-05-04 10:14:37,416 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: status
+2024-05-04 10:14:37,442 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: status
+2024-05-04 10:14:37,443 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: header
+2024-05-04 10:14:37,443 INFO    WriterThread:3810790 [datastore.py:open_for_write():77] open: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/run-st2la7vr.wandb
+2024-05-04 10:14:37,444 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: run
+2024-05-04 10:14:37,445 INFO    SenderThread:3810790 [tracking_sender.py:_maybe_setup_resume():588] checking resume status for None/verifier-reproduce/st2la7vr
+2024-05-04 10:14:38,259 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: check_version
+2024-05-04 10:14:38,262 INFO    SenderThread:3810790 [dir_watcher.py:__init__():216] watching files in: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files
+2024-05-04 10:14:38,262 INFO    SenderThread:3810790 [tracking_sender.py:_start_run_threads():934] run started: st2la7vr with start time 1714788877.412507
+2024-05-04 10:14:38,262 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:14:38,262 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:14:38,262 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: check_version
+2024-05-04 10:14:39,263 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:14:43,263 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: run_start
+2024-05-04 10:14:43,305 DEBUG   HandlerThread:3810790 [system_info.py:__init__():31] System info init
+2024-05-04 10:14:43,305 DEBUG   HandlerThread:3810790 [system_info.py:__init__():46] System info init done
+2024-05-04 10:14:43,305 INFO    HandlerThread:3810790 [system_monitor.py:start():150] Starting system monitor
+2024-05-04 10:14:43,306 INFO    SystemMonitor:3810790 [system_monitor.py:_start():116] Starting system asset monitoring threads
+2024-05-04 10:14:43,306 INFO    HandlerThread:3810790 [system_monitor.py:probe():168] Collecting system info
+2024-05-04 10:14:43,306 INFO    SystemMonitor:3810790 [interfaces.py:start():168] Started cpu
+2024-05-04 10:14:43,307 INFO    SystemMonitor:3810790 [interfaces.py:start():168] Started disk
+2024-05-04 10:14:43,307 INFO    SystemMonitor:3810790 [interfaces.py:start():168] Started gpu
+2024-05-04 10:14:43,307 INFO    SystemMonitor:3810790 [interfaces.py:start():168] Started memory
+2024-05-04 10:14:43,308 INFO    SystemMonitor:3810790 [interfaces.py:start():168] Started network
+2024-05-04 10:14:43,442 DEBUG   HandlerThread:3810790 [system_info.py:probe():195] Probing system
+2024-05-04 10:14:43,448 DEBUG   HandlerThread:3810790 [git.py:repo():40] git repository is invalid
+2024-05-04 10:14:43,448 DEBUG   HandlerThread:3810790 [system_info.py:probe():241] Probing system done
+2024-05-04 10:14:43,448 DEBUG   HandlerThread:3810790 [system_monitor.py:probe():177] {'os': 'Linux-5.10.135.bsk.6-amd64-x86_64-with-glibc2.31', 'python': '3.9.2', 'heartbeatAt': '2024-05-04T02:14:43.442670', 'startedAt': '2024-05-04T02:14:37.405615', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path', '../models/lean4_random_15k_all/checkpoint-236/', '--data_dir', '../repl/pass_rate_results/lean4_15k_train/lean4_random_15k_allpass10.jsonl', '--data_id', 'lean4test', '--target_set', 'train', '--save_dir', '../models/lean4_random_15k_all-sample10-osv-gt', '--generator_id', 'lean4_random_15k_all', '--verifier_id', 'osv', '--dedup', 'True', '--per_problem_sampling_solution', '10', '--loss_level', 'token', '--loss_on_llm', 'True', '--num_train_epoches', '1', '--eval_steps', '10000', '--per_device_train_batch_size', '16', '--per_device_eval_batch_size', '64', '--gradient_accumulation_steps', '4', '--gradient_checkpointing', 'True', '--learning_rate', '2e-6', '--weight_decay', '0', '--save_steps', '1000', '--lr_scheduler_type', 'linear', '--warmup_steps', '0', '--save_epoches', '1', '--save_best', 'False', '--save_total_limit', '0', '--logging_dir', 'None', '--logging_steps', '1', '--easy', 'True', '--seed', '42'), 'state': 'running', 'program': '/opt/tiger/auto-info/PSV/train_psv.py', 'codePath': 'PSV/train_psv.py', 'host': 'n122-200-048', 'username': 'tiger', 'executable': '/usr/bin/python3', 'cpu_count': 90, 'cpu_count_logical': 180, 'cpu_freq': {'current': 2599.5619999999913, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}, {'current': 2599.562, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 3519.748878479004, 'used': 492.3821258544922}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2932.9151611328125}}
+2024-05-04 10:14:43,448 INFO    HandlerThread:3810790 [system_monitor.py:probe():178] Finished collecting system info
+2024-05-04 10:14:43,448 INFO    HandlerThread:3810790 [system_monitor.py:probe():181] Publishing system info
+2024-05-04 10:14:43,448 DEBUG   HandlerThread:3810790 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2024-05-04 10:14:43,449 DEBUG   HandlerThread:3810790 [system_info.py:_save_pip():67] Saving pip packages done
+2024-05-04 10:14:43,450 INFO    HandlerThread:3810790 [system_monitor.py:probe():183] Finished publishing system info
+2024-05-04 10:14:43,485 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: files
+2024-05-04 10:14:43,485 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-metadata.json with policy now
+2024-05-04 10:14:43,488 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:14:43,488 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:14:43,488 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: telemetry
+2024-05-04 10:14:43,582 INFO    Thread-17 :3810790 [upload_job.py:push():143] Uploaded file /tmp/tmpvovo63p4wandb/rb1nz8tp-wandb-metadata.json
+2024-05-04 10:14:44,264 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-metadata.json
+2024-05-04 10:14:44,265 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/requirements.txt
+2024-05-04 10:14:44,265 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_created():275] file/dir created: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:14:46,265 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:14:58,491 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:14:58,491 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:15:02,269 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:15:08,270 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/config.yaml
+2024-05-04 10:15:13,493 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:15:13,494 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:15:22,099 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:15:24,275 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:15:28,496 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:15:28,496 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:15:30,881 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:15:30,882 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:15:30,883 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:15:30,883 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:15:31,276 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:15:32,277 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:15:40,868 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:15:40,869 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:15:40,869 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:15:40,871 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:15:41,279 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:15:42,280 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:15:43,308 DEBUG   SystemMonitor:3810790 [system_monitor.py:_start():130] Starting system metrics aggregation loop
+2024-05-04 10:15:43,311 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: stats
+2024-05-04 10:15:43,497 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:15:43,498 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:15:52,282 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:15:58,499 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:15:58,500 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:16:13,077 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:16:13,078 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:16:13,078 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:16:13,079 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:16:13,288 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:16:13,312 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: stats
+2024-05-04 10:16:13,501 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:16:13,501 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:16:14,288 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:16:24,192 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:16:24,193 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:16:24,193 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:16:24,194 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:16:24,291 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:16:26,291 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:16:28,503 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:16:28,504 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:16:35,778 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:16:35,779 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:16:35,779 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:16:35,780 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:16:36,294 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:16:38,294 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:16:43,314 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: stats
+2024-05-04 10:16:43,506 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:16:43,506 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:16:54,299 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:16:58,508 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:16:58,508 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:17:02,316 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:17:02,317 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:17:02,317 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:17:02,318 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:17:03,301 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:17:04,301 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:17:13,316 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: stats
+2024-05-04 10:17:13,510 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:17:13,510 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:17:14,741 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: partial_history
+2024-05-04 10:17:14,742 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: history
+2024-05-04 10:17:14,742 DEBUG   SenderThread:3810790 [tracking_sender.py:send():315] send: summary
+2024-05-04 10:17:14,743 INFO    SenderThread:3810790 [tracking_sender.py:_save_file():1177] saving file wandb-summary.json with policy end
+2024-05-04 10:17:15,304 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/wandb-summary.json
+2024-05-04 10:17:16,305 INFO    Thread-15 :3810790 [dir_watcher.py:_on_file_modified():292] file/dir modified: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/files/output.log
+2024-05-04 10:17:28,512 DEBUG   HandlerThread:3810790 [handler.py:handle_request():140] handle_request: stop_status
+2024-05-04 10:17:28,512 DEBUG   SenderThread:3810790 [tracking_sender.py:send_request():329] send_request: stop_status
+2024-05-04 10:17:42,603 WARNING StreamThr :3810790 [internal.py:is_dead():385] Internal process exiting, parent pid 3763620 disappeared
+2024-05-04 10:17:42,603 ERROR   StreamThr :3810790 [internal.py:wandb_internal():147] Internal process shutdown.
+2024-05-04 10:17:42,745 INFO    WriterThread:3810790 [datastore.py:close():283] close: None/verifier-reproduce/wandb/run-20240504_101437-st2la7vr/run-st2la7vr.wandb
+2024-05-04 10:17:43,291 INFO    MainThread:3810790 [internal.py:handle_exit():78] Internal process exited
+2024-05-04 10:17:43,292 WARNING MsgRouterThr:3810790 [router.py:message_loop():74] EOFError seen in message_loop