add CLIC training

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/.gitattributes

@@ -0,0 +1,16 @@
+attn_conv_id_2.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_id_5.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_0.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_id_1.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_3.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_5.npz filter=lfs diff=lfs merge=lfs -text
+checkpoints/checkpoint-20-1.914489.pth filter=lfs diff=lfs merge=lfs -text
+preds_checkpoint-20-1.914489 filter=lfs diff=lfs merge=lfs -text
+runs filter=lfs diff=lfs merge=lfs -text
+attn_conv_id_0.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_id_3.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_id_4.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_1.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_2.npz filter=lfs diff=lfs merge=lfs -text
+attn_conv_reg_4.npz filter=lfs diff=lfs merge=lfs -text
+plots_checkpoint-20-1.914489/ filter=lfs diff=lfs merge=lfs -text

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/README.md

@@ -0,0 +1,166 @@
+# Model Card for mlpf-clic-clusters-v2.1.0
+
+This model reconstructs particles in a detector, based on the tracks and calorimeter clusters recorded by the detector.
+
+## Model Details
+
+The performance is measured with respect to generator-level jets and MET computed from Pythia particles, i.e. the truth-level jets and MET.
+
+<details>
+  <summary>Jet performance</summary>
+  
+  <img src="plots_checkpoint-20-1.914489/clic_edm_ttbar_pf/jet_response_iqr_over_med_pt.png" alt="ttbar jet resolution" width="300"/>
+  <img src="plots_checkpoint-20-1.914489/clic_edm_qq_pf/jet_response_iqr_over_med_pt.png" alt="qq jet resolution" width="300"/>
+  <img src="plots_checkpoint-20-1.914489/clic_edm_ww_fullhad_pf/jet_response_iqr_over_med_pt.png" alt="ttbar jet resolution" width="300"/>
+
+</details>
+
+<details>
+  <summary>MET performance</summary>
+  
+  <img src="plots_checkpoint-20-1.914489/clic_edm_ttbar_pf/met_response_iqr_over_med.png" alt="ttbar MET resolution" width="300"/>
+  <img src="plots_checkpoint-20-1.914489/clic_edm_qq_pf/met_response_iqr_over_med.png" alt="qq MET resolution" width="300"/>
+  <img src="plots_checkpoint-20-1.914489/clic_edm_ww_fullhad_pf/met_response_iqr_over_med.png" alt="ttbar MET resolution" width="300"/>
+
+</details>
+
+### Model Description
+
+- **Developed by:** Joosep Pata, Eric Wulff, Farouk Mokhtar, Mengke Zhang, David Southwick, Maria Girone, David Southwick, Javier Duarte, Michael Kagan
+- **Model type:** transformer
+- **License:** Apache License
+
+### Model Sources
+
+- **Repository:** https://github.com/jpata/particleflow/releases/tag/v2.1.0
+
+## Uses
+### Direct Use
+
+This model may be used to study the physics and computational performance on ML-based reconstruction in simulation.
+
+### Out-of-Scope Use
+
+This model is not intended for physics measurements on real data. 
+
+## Bias, Risks, and Limitations
+
+The model has only been trained on simulation data and has not been validated against real data.
+The model has not been peer reviewed or published in a peer-reviewed journal.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+```
+#get the code
+git clone https://github.com/jpata/particleflow
+cd particleflow
+git checkout v2.1.0
+
+#get the models
+git clone https://huggingface.co/jpata/particleflow models
+```
+
+## Training Details
+Trained on 8x MI250X for 26 epochs over ~5 days.
+The training was continued several times from a checkpoint due to a runtime limit.
+
+### Training Data
+The following datasets were used:
+```
+47G	/local/joosep/mlpf/tensorflow_datasets/clic/clic_edm_qq_pf/2.5.0
+93G	/local/joosep/mlpf/tensorflow_datasets/clic/clic_edm_ttbar_pf/2.5.0
+74G	/local/joosep/mlpf/tensorflow_datasets/clic/clic_edm_ww_fullhad_pf/2.5.0
+```
+
+The datasets were generated using Key4HEP with the following scripts:
+- https://github.com/HEP-KBFI/key4hep-sim/releases/tag/v1.1.0
+- https://github.com/HEP-KBFI/key4hep-sim/blob/v1.1.0/clic/run_sim.sh
+
+## Training Procedure 
+
+```bash
+#!/bin/bash
+#SBATCH --job-name=mlpf-train
+#SBATCH --account=project_465000301
+#SBATCH --time=3-00:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=200G
+#SBATCH --gpus-per-task=8
+#SBATCH --partition=small-g
+#SBATCH --no-requeue
+#SBATCH -o logs/slurm-%x-%j-%N.out
+
+cd /scratch/project_465000301/particleflow
+
+module load LUMI/24.03 partition/G
+
+export IMG=/scratch/project_465000301/pytorch-rocm6.2.simg
+export PYTHONPATH=`pwd`
+export TFDS_DATA_DIR=/scratch/project_465000301/tensorflow_datasets
+#export MIOPEN_DISABLE_CACHE=true
+export MIOPEN_USER_DB_PATH=/tmp/${USER}-${SLURM_JOB_ID}-miopen-cache
+export MIOPEN_CUSTOM_CACHE_DIR=${MIOPEN_USER_DB_PATH}
+export TF_CPP_MAX_VLOG_LEVEL=-1 #to suppress ROCm fusion is enabled messages
+export ROCM_PATH=/opt/rocm
+#export NCCL_DEBUG=INFO
+#export MIOPEN_ENABLE_LOGGING=1
+#export MIOPEN_ENABLE_LOGGING_CMD=1
+#export MIOPEN_LOG_LEVEL=4
+export KERAS_BACKEND=torch
+
+env
+
+#TF training
+singularity exec \
+    --rocm \
+    -B /scratch/project_465000301 \
+    -B /tmp \
+    --env LD_LIBRARY_PATH=/opt/rocm/lib/ \
+    --env CUDA_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES \
+     $IMG python3 mlpf/pipeline.py --gpus 8 \
+     --data-dir $TFDS_DATA_DIR --config parameters/pytorch/pyg-clic.yaml \
+     --train --gpu-batch-multiplier 128 --num-workers 8 --prefetch-factor 100 --checkpoint-freq 1 --conv-type attention --dtype bfloat16 --lr 0.0001 --num-epochs 50
+```
+
+## Evaluation
+```bash
+#!/bin/bash
+#SBATCH --partition gpu
+#SBATCH --gres gpu:mig:1
+#SBATCH --mem-per-gpu 200G
+#SBATCH -o logs/slurm-%x-%j-%N.out
+
+IMG=/home/software/singularity/pytorch.simg:2024-08-18
+cd ~/particleflow
+
+WEIGHTS=experiments/pyg-clic_20241106_104416_929167/checkpoints/checkpoint-20-1.914489.pth
+singularity exec -B /scratch/persistent --nv \
+     --env PYTHONPATH=`pwd` \
+     --env KERAS_BACKEND=torch \
+     $IMG  python3 mlpf/pipeline.py --gpus 1 \
+     --data-dir /scratch/persistent/joosep/tensorflow_datasets --config parameters/pytorch/pyg-clic.yaml \
+     --test --make-plots --gpu-batch-multiplier 100 --load $WEIGHTS --dtype bfloat16 --prefetch-factor 10 --num-workers 8 --ntest 50000
+```
+
+
+## Citation
+
+## Glossary
+
+- PF: particle flow reconstruction
+- MLPF: machine learning for particle flow
+- CLIC: Compact Linear Collider
+
+## Model Card Contact
+
+Joosep Pata, [email protected]
+
+## Full outputs
+
+```
+/local/joosep/mlpf/results/clic/pyg-clic_20241106_104416_929167
+```

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:273e622b4eb8ff1ebec6cbfd0cb614367e1a758a73ed9f72bf863fa34ab39014
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a35aa6f579bae96881495b168d4fdeb113670be70730a79c65bbf68b1bc27754
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:660c8b004522012bfbaa9474689ef82ab661ecca90c0fc60d7ab62b8c5fe8bb4
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_3.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43440763503a7ab9f9c19b990ce2d2ba16b70b5d2875eba7c25b0f8a11e3784
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_4.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4381506dd05ede2be3bd0ddabdc96b17c1c60a00157145aabea30554e6613f6
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_id_5.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e627d06392ce38a7e02efa640c9fc27828a52e8fb2fe6d506b22ec8cae2d8bdb
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc98017b9dc71567c1fce4b5b575901670de5fae22ba7e494901a7a632e7723
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a10b1bd04523b88f3d4d3d3f38e5cba7a55d0041985d8acd5a33cfb4200eb12
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7b8874aaf9064aeb73754297837156732747d79a5315528e39047ea45d2ca91
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_3.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ed1e340a2a41021b30f1530f9122fe50492dfcbd35c3ba2bcffffd457f6694
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_4.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cfb87da457664c720a3bf6e6036923c750ab95701de7eec81f84a4d05994938
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/attn_conv_reg_5.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f716509481db992552258d10bd7ead3db240871eee05f5df50452d5114cd798b
+size 27380232

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/best_weights.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee380b1469e20f66c32cd0b48e010cd2b353e01e6db0c17b74cb01750b584154
+size 1072932830

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/checkpoints/checkpoint-20-1.914489.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384a0cc7afad7112063d651e007a62c493a7aabd726bf2010cef56bc92697c45
+size 1072941974

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_1.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.2094265004264538, "Regression_eta": 0.0002604969489546658, "Regression_sin_phi": 0.0001727235758788099, "Regression_cos_phi": 0.000172586845980919, "Regression_energy": 0.20407082290725595, "Classification_binary": 2.82970878554277, "Classification": 0.025728516739426322, "MET": 6.941960677095715, "Sliced_Wasserstein_Loss": 44.39020374684989, "Total": 3.2695373793447766}, "valid": {"Regression_pt": 0.1652884156736609, "Regression_eta": 0.00020157147760260595, "Regression_sin_phi": 0.00014097294578813526, "Regression_cos_phi": 0.0001405951094953981, "Regression_energy": 0.16063081401668183, "Classification_binary": 2.3603847973967254, "Classification": 0.01873019623429808, "MET": 4.846416316620291, "Sliced_Wasserstein_Loss": 39.17741331335616, "Total": 2.705516658417166}, "epoch_train_time": 16789.783430337906, "epoch_valid_time": 769.6228659152985, "epoch_total_time": 17566.47555398941}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_10.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11557132644221625, "Regression_eta": 0.0001308017311896097, "Regression_sin_phi": 8.440069281410419e-05, "Regression_cos_phi": 8.410228131404759e-05, "Regression_energy": 0.11125179840547882, "Classification_binary": 1.7208206041082212, "Classification": 0.010844219904824277, "MET": 4.174885436973991, "Sliced_Wasserstein_Loss": 33.151021111692266, "Total": 1.9587859142456374}, "valid": {"Regression_pt": 0.11739337607605817, "Regression_eta": 0.00013587298509601045, "Regression_sin_phi": 8.860706313423914e-05, "Regression_cos_phi": 8.892184378553744e-05, "Regression_energy": 0.11370478590873823, "Classification_binary": 1.7518208124866224, "Classification": 0.010785341262817383, "MET": 4.04153944041631, "Sliced_Wasserstein_Loss": 33.122361274614725, "Total": 1.994017927613977}, "epoch_train_time": 17259.76031780243, "epoch_valid_time": 623.9745128154755, "epoch_total_time": 17889.780810832977}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_11.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11442679899983953, "Regression_eta": 0.00012909314595118586, "Regression_sin_phi": 8.334942230410348e-05, "Regression_cos_phi": 8.314223019907044e-05, "Regression_energy": 0.11015823275016642, "Classification_binary": 1.70715197951833, "Classification": 0.010682826885386453, "MET": 4.153619953877609, "Sliced_Wasserstein_Loss": 33.00037741904807, "Total": 1.9427162581427417}, "valid": {"Regression_pt": 0.11606519516200235, "Regression_eta": 0.00013497989777832816, "Regression_sin_phi": 8.751226793209167e-05, "Regression_cos_phi": 8.717211830901773e-05, "Regression_energy": 0.11243655583629869, "Classification_binary": 1.7416976510661921, "Classification": 0.010666612892934722, "MET": 4.020172955238656, "Sliced_Wasserstein_Loss": 32.966796875, "Total": 1.9811754618605522}, "epoch_train_time": 17172.525350809097, "epoch_valid_time": 623.3843004703522, "epoch_total_time": 17800.964765548706}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_12.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11337792648352432, "Regression_eta": 0.00012755816834532162, "Regression_sin_phi": 8.243371402932203e-05, "Regression_cos_phi": 8.219390105693296e-05, "Regression_energy": 0.10915779134447363, "Classification_binary": 1.6955058529670486, "Classification": 0.010549265502505973, "MET": 4.1333734279159335, "Sliced_Wasserstein_Loss": 32.86684537111883, "Total": 1.9288832556702011}, "valid": {"Regression_pt": 0.11523698780634632, "Regression_eta": 0.00013245166317649084, "Regression_sin_phi": 8.66244687405351e-05, "Regression_cos_phi": 8.658088156825876e-05, "Regression_energy": 0.1115259144404163, "Classification_binary": 1.7325731042313248, "Classification": 0.010522361487558442, "MET": 4.002341492535317, "Sliced_Wasserstein_Loss": 32.86091676476884, "Total": 1.9701634237211045}, "epoch_train_time": 17151.1980175972, "epoch_valid_time": 621.2587215900421, "epoch_total_time": 17777.9982585907}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_13.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11202425836785579, "Regression_eta": 0.00012615428638154433, "Regression_sin_phi": 8.159382447248223e-05, "Regression_cos_phi": 8.134114453764344e-05, "Regression_energy": 0.10786196892013575, "Classification_binary": 1.6851392735723456, "Classification": 0.01043629851492477, "MET": 4.114489845347344, "Sliced_Wasserstein_Loss": 32.73006811373686, "Total": 1.9157512273548571}, "valid": {"Regression_pt": 0.11388364556717546, "Regression_eta": 0.00013183363496440732, "Regression_sin_phi": 8.560803221309021e-05, "Regression_cos_phi": 8.558747932723123e-05, "Regression_energy": 0.11024029614174202, "Classification_binary": 1.7250676612331444, "Classification": 0.010420655551022047, "MET": 3.9703519638270546, "Sliced_Wasserstein_Loss": 32.701877541738014, "Total": 1.9599159972308433}, "epoch_train_time": 17202.07878088951, "epoch_valid_time": 622.1237692832947, "epoch_total_time": 17830.736182689667}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_14.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.1107143895284045, "Regression_eta": 0.00012473149508440478, "Regression_sin_phi": 8.067128593789817e-05, "Regression_cos_phi": 8.046297151631667e-05, "Regression_energy": 0.10658827250175336, "Classification_binary": 1.6757981521325662, "Classification": 0.010336491335056732, "MET": 4.092183116589796, "Sliced_Wasserstein_Loss": 32.59393870952404, "Total": 1.9037224810993296}, "valid": {"Regression_pt": 0.11284738044216208, "Regression_eta": 0.00013056410195892804, "Regression_sin_phi": 8.495507940445861e-05, "Regression_cos_phi": 8.47419149765413e-05, "Regression_energy": 0.10922682775210028, "Classification_binary": 1.7167156689787564, "Classification": 0.010336691386079134, "MET": 3.970846515812286, "Sliced_Wasserstein_Loss": 32.578305597174655, "Total": 1.9494273564586901}, "epoch_train_time": 17200.5848031044, "epoch_valid_time": 615.1532604694366, "epoch_total_time": 17822.41261768341}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_15.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.109766175247997, "Regression_eta": 0.00012358875010743767, "Regression_sin_phi": 7.989672880801681e-05, "Regression_cos_phi": 7.968661754630464e-05, "Regression_energy": 0.10566881274220198, "Classification_binary": 1.667280406007798, "Classification": 0.01025230353662265, "MET": 4.071760139793638, "Sliced_Wasserstein_Loss": 32.484439636726734, "Total": 1.8932520743188626}, "valid": {"Regression_pt": 0.11216124443158712, "Regression_eta": 0.00012894860175374437, "Regression_sin_phi": 8.412625972334652e-05, "Regression_cos_phi": 8.410891543512475e-05, "Regression_energy": 0.10862473918967051, "Classification_binary": 1.7108230590820312, "Classification": 0.010261405820715917, "MET": 3.9149236809717465, "Sliced_Wasserstein_Loss": 32.47261611729452, "Total": 1.9421683533550942}, "epoch_train_time": 17199.00596857071, "epoch_valid_time": 620.6472885608673, "epoch_total_time": 17826.198667287827}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_16.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10893874852895725, "Regression_eta": 0.0001225375589531525, "Regression_sin_phi": 7.920775681735502e-05, "Regression_cos_phi": 7.90164301871664e-05, "Regression_energy": 0.10486779560529694, "Classification_binary": 1.6595828776567922, "Classification": 0.010179955066795, "MET": 4.056352452926632, "Sliced_Wasserstein_Loss": 32.381157576910276, "Total": 1.8838481735295516}, "valid": {"Regression_pt": 0.11124360071469659, "Regression_eta": 0.00012906229965490838, "Regression_sin_phi": 8.326997240520503e-05, "Regression_cos_phi": 8.343247543663194e-05, "Regression_energy": 0.10758737668599168, "Classification_binary": 1.7061379837663206, "Classification": 0.010203114927631535, "MET": 3.930666152745077, "Sliced_Wasserstein_Loss": 32.38690269156678, "Total": 1.9354682817851028}, "epoch_train_time": 17198.147196531296, "epoch_valid_time": 615.1392314434052, "epoch_total_time": 17819.922924280167}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_17.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10818741623228804, "Regression_eta": 0.00012158320444489868, "Regression_sin_phi": 7.86403577829143e-05, "Regression_cos_phi": 7.844559051563009e-05, "Regression_energy": 0.10414024934546028, "Classification_binary": 1.6524665226332558, "Classification": 0.010115624517081926, "MET": 4.034255607793257, "Sliced_Wasserstein_Loss": 32.28857995340212, "Total": 1.875188709524036}, "valid": {"Regression_pt": 0.11076493459205104, "Regression_eta": 0.00012701347929565873, "Regression_sin_phi": 8.289593794982727e-05, "Regression_cos_phi": 8.275241858951033e-05, "Regression_energy": 0.10716541499307711, "Classification_binary": 1.7020633645253638, "Classification": 0.010158186089502622, "MET": 3.8972151246789384, "Sliced_Wasserstein_Loss": 32.345970676369866, "Total": 1.930444168717894}, "epoch_train_time": 17223.179953813553, "epoch_valid_time": 627.4559621810913, "epoch_total_time": 17857.377381324768}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_18.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10750404072410964, "Regression_eta": 0.00012067331181041389, "Regression_sin_phi": 7.808604110500838e-05, "Regression_cos_phi": 7.792408742610366e-05, "Regression_energy": 0.10348230829638391, "Classification_binary": 1.645835128797965, "Classification": 0.010057561889898023, "MET": 4.018934474704008, "Sliced_Wasserstein_Loss": 32.20713470591032, "Total": 1.8671536957348676}, "valid": {"Regression_pt": 0.1102727080044681, "Regression_eta": 0.00012721782167480418, "Regression_sin_phi": 8.21975098080831e-05, "Regression_cos_phi": 8.231317241714425e-05, "Regression_energy": 0.10668338488226067, "Classification_binary": 1.697039251458155, "Classification": 0.01010439983785969, "MET": 3.8683689065175515, "Sliced_Wasserstein_Loss": 32.28075168557363, "Total": 1.9243918640972817}, "epoch_train_time": 17275.312012434006, "epoch_valid_time": 614.2752957344055, "epoch_total_time": 17896.61068558693}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_19.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.1068745988065237, "Regression_eta": 0.00011987102552621354, "Regression_sin_phi": 7.758405385324162e-05, "Regression_cos_phi": 7.741430630941692e-05, "Regression_energy": 0.10287435391133921, "Classification_binary": 1.6395421490894393, "Classification": 0.010004844300944513, "MET": 4.003260811421235, "Sliced_Wasserstein_Loss": 32.122366982074084, "Total": 1.8595692816556513}, "valid": {"Regression_pt": 0.1098469904024307, "Regression_eta": 0.0001269323266532323, "Regression_sin_phi": 8.185888474730596e-05, "Regression_cos_phi": 8.181145746413975e-05, "Regression_energy": 0.10627154781393809, "Classification_binary": 1.6933489237746147, "Classification": 0.0100542862121373, "MET": 3.943997317797517, "Sliced_Wasserstein_Loss": 32.137494648972606, "Total": 1.9198127642069778}, "epoch_train_time": 17292.59947705269, "epoch_valid_time": 630.3664655685425, "epoch_total_time": 17929.433540582657}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_2.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.15229772187113666, "Regression_eta": 0.00017695347358471803, "Regression_sin_phi": 0.00012325244058494103, "Regression_cos_phi": 0.0001239232004388843, "Regression_energy": 0.1470453377417562, "Classification_binary": 2.189257078836004, "Classification": 0.016820775091401532, "MET": 4.7845966222481096, "Sliced_Wasserstein_Loss": 37.977479672863865, "Total": 2.505842565736294}, "valid": {"Regression_pt": 0.14605353629752382, "Regression_eta": 0.0001744219956740941, "Regression_sin_phi": 0.0001195611253584901, "Regression_cos_phi": 0.00011941437188484897, "Regression_energy": 0.1413056491172477, "Classification_binary": 2.127994903146404, "Classification": 0.015635341814119522, "MET": 4.563908825181935, "Sliced_Wasserstein_Loss": 36.76232408497432, "Total": 2.4314044273062927}, "epoch_train_time": 16800.862768173218, "epoch_valid_time": 613.5964965820312, "epoch_total_time": 17421.3773291111}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_3.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.13842518048134303, "Regression_eta": 0.00016091593287594335, "Regression_sin_phi": 0.00010859709505653716, "Regression_cos_phi": 0.00010902233966310266, "Regression_energy": 0.13320921255378965, "Classification_binary": 2.0393329341210595, "Classification": 0.015051829356348218, "MET": 4.550837900004755, "Sliced_Wasserstein_Loss": 36.33040214445342, "Total": 2.326395224608911}, "valid": {"Regression_pt": 0.1364705595251632, "Regression_eta": 0.0001641854975525647, "Regression_sin_phi": 0.00010914304485059764, "Regression_cos_phi": 0.00010963623756415223, "Regression_energy": 0.13192350570469685, "Classification_binary": 2.0156960683326197, "Classification": 0.014360421324429447, "MET": 4.36081434276006, "Sliced_Wasserstein_Loss": 35.720074379280824, "Total": 2.2988329325636774}, "epoch_train_time": 16699.64409685135, "epoch_valid_time": 635.7045834064484, "epoch_total_time": 17341.40076160431}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_4.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.13159020649576816, "Regression_eta": 0.00015296980701087572, "Regression_sin_phi": 0.00010190477816061288, "Regression_cos_phi": 0.00010189606036843329, "Regression_energy": 0.12645080894349411, "Classification_binary": 1.9457036636393894, "Classification": 0.013780934965608803, "MET": 4.4381062479197375, "Sliced_Wasserstein_Loss": 35.407172744995485, "Total": 2.2178852051733156}, "valid": {"Regression_pt": 0.13098476357655983, "Regression_eta": 0.0001566166262308212, "Regression_sin_phi": 0.00010432322088577976, "Regression_cos_phi": 0.00010397419818256, "Regression_energy": 0.12651298470693093, "Classification_binary": 1.9377539647768622, "Classification": 0.013109352490673326, "MET": 4.18827798921768, "Sliced_Wasserstein_Loss": 35.13144129922945, "Total": 2.2087262297329837}, "epoch_train_time": 16704.854731082916, "epoch_valid_time": 633.9079310894012, "epoch_total_time": 17346.477849960327}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_5.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.1266045301244294, "Regression_eta": 0.00014589743292012572, "Regression_sin_phi": 9.584324851105678e-05, "Regression_cos_phi": 9.546951710041338e-05, "Regression_energy": 0.12198213351718891, "Classification_binary": 1.8743963524083496, "Classification": 0.012682432749943537, "MET": 4.3794747931624745, "Sliced_Wasserstein_Loss": 34.724772359849744, "Total": 2.136000130759355}, "valid": {"Regression_pt": 0.12681689327710297, "Regression_eta": 0.0001496519536188204, "Regression_sin_phi": 9.81583885134083e-05, "Regression_cos_phi": 9.769133662116037e-05, "Regression_energy": 0.12270259857177734, "Classification_binary": 1.876480520588078, "Classification": 0.012277160605339154, "MET": 4.126168865047089, "Sliced_Wasserstein_Loss": 34.58176704302226, "Total": 2.1386210820446276}, "epoch_train_time": 24272.22338628769, "epoch_valid_time": 621.7568323612213, "epoch_total_time": 24900.24202799797}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_6.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.123127519996523, "Regression_eta": 0.00014168217067547098, "Regression_sin_phi": 9.18078970100824e-05, "Regression_cos_phi": 9.144574586564376e-05, "Regression_energy": 0.11861148145148828, "Classification_binary": 1.8210901838357663, "Classification": 0.012012854797354575, "MET": 4.317918340782654, "Sliced_Wasserstein_Loss": 34.21404117730969, "Total": 2.0751665695877515}, "valid": {"Regression_pt": 0.12352200077004628, "Regression_eta": 0.00014717310462912468, "Regression_sin_phi": 9.508110735922644e-05, "Regression_cos_phi": 9.477458103266481e-05, "Regression_energy": 0.11949882768604854, "Classification_binary": 1.8318142825610018, "Classification": 0.011771371103312871, "MET": 4.088931148999358, "Sliced_Wasserstein_Loss": 34.12710696703767, "Total": 2.086944789102633}, "epoch_train_time": 17158.664612293243, "epoch_valid_time": 623.92746925354, "epoch_total_time": 17789.146901130676}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_7.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.12053723335492844, "Regression_eta": 0.0001382408760610964, "Regression_sin_phi": 8.923594425954705e-05, "Regression_cos_phi": 8.886856697441298e-05, "Regression_energy": 0.11607258611543673, "Classification_binary": 1.7843677933764444, "Classification": 0.011596784164967785, "MET": 4.271511771313775, "Sliced_Wasserstein_Loss": 33.84363261376064, "Total": 2.0328861635799536}, "valid": {"Regression_pt": 0.12122258748093696, "Regression_eta": 0.0001426156712313221, "Regression_sin_phi": 9.291608218256741e-05, "Regression_cos_phi": 9.233869408091454e-05, "Regression_energy": 0.11753453293891802, "Classification_binary": 1.8036887809021833, "Classification": 0.011456087027510551, "MET": 4.103804914918665, "Sliced_Wasserstein_Loss": 33.780450690282535, "Total": 2.054229527303617}, "epoch_train_time": 17156.06824874878, "epoch_valid_time": 622.7183349132538, "epoch_total_time": 17785.415353298187}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_8.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11850913996748846, "Regression_eta": 0.00013524123001334434, "Regression_sin_phi": 8.720222731798817e-05, "Regression_cos_phi": 8.688842080911506e-05, "Regression_energy": 0.11409453743506609, "Classification_binary": 1.7577993126218439, "Classification": 0.011291456746281903, "MET": 4.232693702153963, "Sliced_Wasserstein_Loss": 33.55871095050164, "Total": 2.0020067103323664}, "valid": {"Regression_pt": 0.1197902209138217, "Regression_eta": 0.00014036432010670232, "Regression_sin_phi": 9.121422413481425e-05, "Regression_cos_phi": 9.099420874494395e-05, "Regression_energy": 0.11594179231826574, "Classification_binary": 1.7847603575824058, "Classification": 0.011192953749878765, "MET": 4.0823602545751285, "Sliced_Wasserstein_Loss": 33.46659286708048, "Total": 2.0320081318894476}, "epoch_train_time": 17171.700482845306, "epoch_valid_time": 629.8177120685577, "epoch_total_time": 17808.156106948853}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/history/epoch_9.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.1168886828335255, "Regression_eta": 0.00013286315461242596, "Regression_sin_phi": 8.565038095430439e-05, "Regression_cos_phi": 8.53920894367232e-05, "Regression_energy": 0.1125250885225976, "Classification_binary": 1.7372543061433123, "Classification": 0.011044098360381699, "MET": 4.202892827849365, "Sliced_Wasserstein_Loss": 33.3332313014122, "Total": 1.9780166406138557}, "valid": {"Regression_pt": 0.11840791571630191, "Regression_eta": 0.00013830808744038622, "Regression_sin_phi": 8.961594063941746e-05, "Regression_cos_phi": 8.949798799745024e-05, "Regression_energy": 0.11483025224241492, "Classification_binary": 1.7646609789704624, "Classification": 0.010979686697868451, "MET": 4.03977401942423, "Sliced_Wasserstein_Loss": 33.21316820954623, "Total": 2.00919624224101}, "epoch_train_time": 17203.122814893723, "epoch_valid_time": 623.9160017967224, "epoch_total_time": 17833.6868622303}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/hyperparameters.json

@@ -0,0 +1 @@
+{"Num of mlpf parameters": 89388050, "config": "parameters/pytorch/pyg-clic.yaml", "prefix": null, "data_dir": "/scratch/project_465000301/tensorflow_datasets", "gpus": 8, "gpu_batch_multiplier": 128, "num_workers": 8, "prefetch_factor": 100, "resume_training": null, "load": "experiments/pyg-clic_20241106_104416_929167/checkpoints/checkpoint-04-2.208726.pth", "train": true, "test": null, "num_epochs": 50, "patience": null, "lr": 0.0001, "conv_type": "attention", "num_convs": null, "make_plots": null, "export_onnx": null, "ntrain": null, "ntest": null, "nvalid": null, "val_freq": null, "checkpoint_freq": 1, "hpo": null, "ray_train": false, "local": null, "ray_cpus": null, "ray_gpus": null, "raytune_num_samples": null, "comet": false, "comet_offline": false, "comet_step_freq": null, "experiments_dir": null, "pipeline": null, "dtype": "bfloat16", "attention_type": null, "test_datasets": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}}

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/mlpf_losses.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afdc2a865198e5e8fc3ac0da2276f1b94e6a5ec02e05956fd38701d711245bb1
+size 1992

clic/clusters/v2.1.0/pyg-clic_20241106_104416_929167/model_kwargs.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9588bea063af38a14af7818140738a2186aeeb5362c416bd61bf0e89d523fe83
+size 552