Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: mit
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: microsoft/phi-2
+model-index:
+- name: phi-2-gpo-test-longest-iter-random2-0
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# phi-2-gpo-test-longest-iter-random2-0
+
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0012
+- Rewards/chosen: -0.0039
+- Rewards/rejected: -0.0042
+- Rewards/accuracies: 0.5060
+- Rewards/margins: 0.0002
+- Logps/rejected: -233.9551
+- Logps/chosen: -257.0081
+- Logits/rejected: 0.8653
+- Logits/chosen: 0.8074
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 4
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0011        | 1.6   | 100  | 0.0012          | -0.0021        | -0.0022          | 0.5050             | 0.0001          | -233.7621      | -256.8261    | 0.8786          | 0.8210        |
+| 0.001         | 3.2   | 200  | 0.0012          | -0.0041        | -0.0040          | 0.5010             | -0.0000         | -233.9424      | -257.0215    | 0.8660          | 0.8083        |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.2.1+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95b26744ec5f3458555950682d89750114b9ac5aafe5ed03479d34536a7ffbbd
+oid sha256:cefffd0b818ed0cea8bdb4bef462b812cb1144053e4b602682c507cefa31e081
 size 41977616

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 3.97,
+    "eval_logits/chosen": 0.8074077367782593,
+    "eval_logits/rejected": 0.8652730584144592,
+    "eval_logps/chosen": -257.0081481933594,
+    "eval_logps/rejected": -233.95510864257812,
+    "eval_loss": 0.001172678079456091,
+    "eval_rewards/accuracies": 0.5059999823570251,
+    "eval_rewards/chosen": -0.003939597401767969,
+    "eval_rewards/margins": 0.00022889903630129993,
+    "eval_rewards/rejected": -0.0041684964671730995,
+    "eval_runtime": 749.2839,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.669,
+    "eval_steps_per_second": 0.667,
+    "train_loss": 0.0010566485528421077,
+    "train_runtime": 4635.3882,
+    "train_samples": 61135,
+    "train_samples_per_second": 0.863,
+    "train_steps_per_second": 0.054
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 3.97,
+    "eval_logits/chosen": 0.8074077367782593,
+    "eval_logits/rejected": 0.8652730584144592,
+    "eval_logps/chosen": -257.0081481933594,
+    "eval_logps/rejected": -233.95510864257812,
+    "eval_loss": 0.001172678079456091,
+    "eval_rewards/accuracies": 0.5059999823570251,
+    "eval_rewards/chosen": -0.003939597401767969,
+    "eval_rewards/margins": 0.00022889903630129993,
+    "eval_rewards/rejected": -0.0041684964671730995,
+    "eval_runtime": 749.2839,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.669,
+    "eval_steps_per_second": 0.667
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.97,
+    "train_loss": 0.0010566485528421077,
+    "train_runtime": 4635.3882,
+    "train_samples": 61135,
+    "train_samples_per_second": 0.863,
+    "train_steps_per_second": 0.054
+}

trainer_state.json

@@ -0,0 +1,412 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.968,
+  "eval_steps": 100,
+  "global_step": 248,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "learning_rate": 2.0000000000000002e-07,
+      "logits/chosen": 0.7459792494773865,
+      "logits/rejected": 0.8918710947036743,
+      "logps/chosen": -76.09617614746094,
+      "logps/rejected": -62.01979064941406,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 2.0000000000000003e-06,
+      "logits/chosen": 0.6576791405677795,
+      "logits/rejected": 0.7277867197990417,
+      "logps/chosen": -80.96837615966797,
+      "logps/rejected": -67.04137420654297,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.0003823735751211643,
+      "rewards/margins": 0.000583351356908679,
+      "rewards/rejected": -0.00020097770902793854,
+      "step": 10
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": 0.6884250044822693,
+      "logits/rejected": 0.7408124804496765,
+      "logps/chosen": -90.96418762207031,
+      "logps/rejected": -76.06710815429688,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -5.052227152191335e-06,
+      "rewards/margins": -0.0004625328874681145,
+      "rewards/rejected": 0.0004574806080199778,
+      "step": 20
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 4.993800445762451e-06,
+      "logits/chosen": 0.753118634223938,
+      "logits/rejected": 0.815566897392273,
+      "logps/chosen": -75.56269836425781,
+      "logps/rejected": -62.7692756652832,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0003265980340074748,
+      "rewards/margins": 0.00027744597173295915,
+      "rewards/rejected": 4.915207318845205e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.944388344834205e-06,
+      "logits/chosen": 0.6737275719642639,
+      "logits/rejected": 0.7539814710617065,
+      "logps/chosen": -99.6846694946289,
+      "logps/rejected": -84.5612564086914,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.0005740473279729486,
+      "rewards/margins": 0.00023269152734428644,
+      "rewards/rejected": 0.00034135582973249257,
+      "step": 40
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.8465431931347904e-06,
+      "logits/chosen": 0.6654059290885925,
+      "logits/rejected": 0.7606115341186523,
+      "logps/chosen": -92.47711181640625,
+      "logps/rejected": -79.58221435546875,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0007205186993815005,
+      "rewards/margins": 0.0004779071605298668,
+      "rewards/rejected": 0.00024261146609205753,
+      "step": 50
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.702203692102539e-06,
+      "logits/chosen": 0.5616599321365356,
+      "logits/rejected": 0.6836065649986267,
+      "logps/chosen": -78.09901428222656,
+      "logps/rejected": -59.116851806640625,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0014349878765642643,
+      "rewards/margins": 0.0005719334003515542,
+      "rewards/rejected": 0.000863054592628032,
+      "step": 60
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 4.514229781074239e-06,
+      "logits/chosen": 0.6593050360679626,
+      "logits/rejected": 0.7536520957946777,
+      "logps/chosen": -76.82511901855469,
+      "logps/rejected": -66.14640808105469,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": 0.0013633746420964599,
+      "rewards/margins": 1.1641532356165829e-11,
+      "rewards/rejected": 0.001363374525681138,
+      "step": 70
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 4.286345970517195e-06,
+      "logits/chosen": 0.7199975252151489,
+      "logits/rejected": 0.7695997953414917,
+      "logps/chosen": -95.65029907226562,
+      "logps/rejected": -79.29694366455078,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.002262389287352562,
+      "rewards/margins": 0.00043650128645822406,
+      "rewards/rejected": 0.0018258880591019988,
+      "step": 80
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 4.023067544670082e-06,
+      "logits/chosen": 0.6014922261238098,
+      "logits/rejected": 0.7544792890548706,
+      "logps/chosen": -85.02667999267578,
+      "logps/rejected": -71.08529663085938,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0019673267379403114,
+      "rewards/margins": 0.0005049472092650831,
+      "rewards/rejected": 0.0014623795868828893,
+      "step": 90
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 3.7296110958116845e-06,
+      "logits/chosen": 0.6225594282150269,
+      "logits/rejected": 0.6867057085037231,
+      "logps/chosen": -84.804931640625,
+      "logps/rejected": -70.72709655761719,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.001966786338016391,
+      "rewards/margins": 0.00019253513892181218,
+      "rewards/rejected": 0.0017742514610290527,
+      "step": 100
+    },
+    {
+      "epoch": 1.6,
+      "eval_logits/chosen": 0.8210488557815552,
+      "eval_logits/rejected": 0.8785973787307739,
+      "eval_logps/chosen": -256.8260803222656,
+      "eval_logps/rejected": -233.76206970214844,
+      "eval_loss": 0.0011570560745894909,
+      "eval_rewards/accuracies": 0.5049999952316284,
+      "eval_rewards/chosen": -0.00211901543661952,
+      "eval_rewards/margins": 0.00011873205221490934,
+      "eval_rewards/rejected": -0.0022377476561814547,
+      "eval_runtime": 840.5697,
+      "eval_samples_per_second": 2.379,
+      "eval_steps_per_second": 0.595,
+      "step": 100
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 3.4117911628292944e-06,
+      "logits/chosen": 0.6016367673873901,
+      "logits/rejected": 0.6993056535720825,
+      "logps/chosen": -87.06333923339844,
+      "logps/rejected": -73.28208923339844,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0032501216046512127,
+      "rewards/margins": 0.0010317874839529395,
+      "rewards/rejected": 0.0022183340042829514,
+      "step": 110
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 3.075905022087675e-06,
+      "logits/chosen": 0.6844900250434875,
+      "logits/rejected": 0.7173089981079102,
+      "logps/chosen": -87.12005615234375,
+      "logps/rejected": -70.78279876708984,
+      "loss": 0.001,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.003416379215195775,
+      "rewards/margins": 0.0013540134532377124,
+      "rewards/rejected": 0.0020623658783733845,
+      "step": 120
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 2.728607913349464e-06,
+      "logits/chosen": 0.6243213415145874,
+      "logits/rejected": 0.6759988069534302,
+      "logps/chosen": -84.78315734863281,
+      "logps/rejected": -69.31483459472656,
+      "loss": 0.001,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.003974142484366894,
+      "rewards/margins": 0.00143245083745569,
+      "rewards/rejected": 0.0025416917633265257,
+      "step": 130
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 2.376781173017589e-06,
+      "logits/chosen": 0.6302226185798645,
+      "logits/rejected": 0.6924747824668884,
+      "logps/chosen": -71.3794174194336,
+      "logps/rejected": -64.13245391845703,
+      "loss": 0.001,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.003757910802960396,
+      "rewards/margins": 0.001318571506999433,
+      "rewards/rejected": 0.0024393394123762846,
+      "step": 140
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 2.0273958875043877e-06,
+      "logits/chosen": 0.6476806402206421,
+      "logits/rejected": 0.7262701988220215,
+      "logps/chosen": -93.0534896850586,
+      "logps/rejected": -75.62432861328125,
+      "loss": 0.001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004165604244917631,
+      "rewards/margins": 0.0014166636392474174,
+      "rewards/rejected": 0.0027489408385008574,
+      "step": 150
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 1.6873747682962393e-06,
+      "logits/chosen": 0.6091146469116211,
+      "logits/rejected": 0.6964036226272583,
+      "logps/chosen": -101.43008422851562,
+      "logps/rejected": -83.43622589111328,
+      "loss": 0.001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.005559985991567373,
+      "rewards/margins": 0.001756802899762988,
+      "rewards/rejected": 0.0038031828589737415,
+      "step": 160
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 1.363454985517803e-06,
+      "logits/chosen": 0.6119644045829773,
+      "logits/rejected": 0.6556802988052368,
+      "logps/chosen": -70.76570129394531,
+      "logps/rejected": -60.847434997558594,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004071240313351154,
+      "rewards/margins": 0.0009927384089678526,
+      "rewards/rejected": 0.0030785012058913708,
+      "step": 170
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 1.062054677808238e-06,
+      "logits/chosen": 0.6605072021484375,
+      "logits/rejected": 0.7174011468887329,
+      "logps/chosen": -83.5826416015625,
+      "logps/rejected": -65.7984619140625,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0050515844486653805,
+      "rewards/margins": 0.0010326830670237541,
+      "rewards/rejected": 0.004018902312964201,
+      "step": 180
+    },
+    {
+      "epoch": 3.04,
+      "learning_rate": 7.891457834794711e-07,
+      "logits/chosen": 0.6303926706314087,
+      "logits/rejected": 0.7238384485244751,
+      "logps/chosen": -81.99962615966797,
+      "logps/rejected": -67.91422271728516,
+      "loss": 0.001,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005765286739915609,
+      "rewards/margins": 0.001600590767338872,
+      "rewards/rejected": 0.004164696671068668,
+      "step": 190
+    },
+    {
+      "epoch": 3.2,
+      "learning_rate": 5.501357126768117e-07,
+      "logits/chosen": 0.6904739141464233,
+      "logits/rejected": 0.7569997310638428,
+      "logps/chosen": -91.30878448486328,
+      "logps/rejected": -78.43734741210938,
+      "loss": 0.001,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.006007182411849499,
+      "rewards/margins": 0.0020701445173472166,
+      "rewards/rejected": 0.0039370376616716385,
+      "step": 200
+    },
+    {
+      "epoch": 3.2,
+      "eval_logits/chosen": 0.8083356618881226,
+      "eval_logits/rejected": 0.8659967184066772,
+      "eval_logps/chosen": -257.021484375,
+      "eval_logps/rejected": -233.9424285888672,
+      "eval_loss": 0.0011878832010552287,
+      "eval_rewards/accuracies": 0.5009999871253967,
+      "eval_rewards/chosen": -0.004072962328791618,
+      "eval_rewards/margins": -3.138141255476512e-05,
+      "eval_rewards/rejected": -0.004041580483317375,
+      "eval_runtime": 749.8692,
+      "eval_samples_per_second": 2.667,
+      "eval_steps_per_second": 0.667,
+      "step": 200
+    },
+    {
+      "epoch": 3.36,
+      "learning_rate": 3.4976020508682345e-07,
+      "logits/chosen": 0.5687087178230286,
+      "logits/rejected": 0.6581428647041321,
+      "logps/chosen": -82.01387023925781,
+      "logps/rejected": -71.76741790771484,
+      "loss": 0.001,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.005611591041088104,
+      "rewards/margins": 0.001832630136050284,
+      "rewards/rejected": 0.003778961021453142,
+      "step": 210
+    },
+    {
+      "epoch": 3.52,
+      "learning_rate": 1.9198949610721273e-07,
+      "logits/chosen": 0.6064807176589966,
+      "logits/rejected": 0.6868988275527954,
+      "logps/chosen": -88.08094787597656,
+      "logps/rejected": -67.92098236083984,
+      "loss": 0.001,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.005776416976004839,
+      "rewards/margins": 0.0024658578913658857,
+      "rewards/rejected": 0.003310559317469597,
+      "step": 220
+    },
+    {
+      "epoch": 3.68,
+      "learning_rate": 7.994965069994143e-08,
+      "logits/chosen": 0.6825326681137085,
+      "logits/rejected": 0.7606233954429626,
+      "logps/chosen": -90.36261749267578,
+      "logps/rejected": -74.85285949707031,
+      "loss": 0.001,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.005841919686645269,
+      "rewards/margins": 0.0014079047832638025,
+      "rewards/rejected": 0.004434015601873398,
+      "step": 230
+    },
+    {
+      "epoch": 3.84,
+      "learning_rate": 1.5860623616664183e-08,
+      "logits/chosen": 0.6195131540298462,
+      "logits/rejected": 0.6464000940322876,
+      "logps/chosen": -86.05440521240234,
+      "logps/rejected": -73.87252044677734,
+      "loss": 0.001,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006215591914951801,
+      "rewards/margins": 0.0018872864311560988,
+      "rewards/rejected": 0.004328305833041668,
+      "step": 240
+    },
+    {
+      "epoch": 3.97,
+      "step": 248,
+      "total_flos": 0.0,
+      "train_loss": 0.0010566485528421077,
+      "train_runtime": 4635.3882,
+      "train_samples_per_second": 0.863,
+      "train_steps_per_second": 0.054
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 248,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}