Training in progress, step 1500

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00576dc9415d630a78c36e6603cc13167b744e86340891a833e9b836c1697077
+oid sha256:6fa01ed62056a0c4963040de3d44a8711f219d2509e9833786372f6566ee7e57
 size 990345064

runs/Feb26_09-23-32_d79522c38279/events.out.tfevents.1708939413.d79522c38279.34.5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84d9ce222a16e2e911d2f9edb83b15f8e86c5d0f5c64748630dcc8b000bdbdba
-size 5753
+oid sha256:66ff0d405bf89528179fb7b9cef4b925039008554ae30c44a0fb2e99ac3d114c
+size 6280

wandb/debug-internal.log

@@ -396,3 +396,77 @@
 2024-02-26 09:28:32,703 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
 2024-02-26 09:28:37,495 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
 2024-02-26 09:28:37,704 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:42,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:42,705 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:47,497 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:47,706 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:52,498 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:52,707 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:57,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:57,712 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:02,495 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:29:02,713 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:03,495 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:07,714 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:08,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:12,715 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:13,497 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:17,716 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:18,498 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:22,717 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:23,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:27,718 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:28,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:32,495 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:29:32,724 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:34,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:37,725 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:39,577 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/output.log
+2024-02-26 09:29:39,706 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:42,726 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:44,707 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:47,728 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:49,708 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:52,729 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:54,708 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:57,750 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:59,709 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:02,496 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:30:02,778 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:02,779 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-26 09:30:02,780 DEBUG SenderThread:196 [sender.py:send():382] send: history
+2024-02-26 09:30:02,781 DEBUG SenderThread:196 [sender.py:send_request():409] send_request: summary_record
+2024-02-26 09:30:02,781 INFO SenderThread:196 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-02-26 09:30:03,587 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/wandb-summary.json
+2024-02-26 09:30:04,782 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:07,779 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:09,783 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:12,781 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:14,783 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:17,782 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:19,784 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:22,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:24,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:27,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:29,786 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:32,497 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:30:32,787 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:33,462 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-26 09:30:33,463 DEBUG SenderThread:196 [sender.py:send():382] send: history
+2024-02-26 09:30:33,463 DEBUG SenderThread:196 [sender.py:send_request():409] send_request: summary_record
+2024-02-26 09:30:33,464 INFO SenderThread:196 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-02-26 09:30:33,596 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/wandb-summary.json
+2024-02-26 09:30:35,464 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:38,367 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:40,465 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:43,368 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:45,466 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:48,369 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:50,467 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:53,370 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:55,470 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:58,371 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:31:00,471 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:31:02,498 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:31:03,372 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:31:05,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240226_092105-91uhs91a/files/output.log

@@ -1,3 +1,5 @@
+/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:1128: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.
+ warnings.warn(
 /opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:1128: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.
  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:1128: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.

wandb/run-20240226_092105-91uhs91a/files/wandb-summary.json

@@ -1 +1 @@
-{"eval/loss": 1.668515920639038, "eval/bleu": 15.1417, "eval/gen_len": 11.7188, "eval/runtime": 26.1739, "eval/samples_per_second": 30.565, "eval/steps_per_second": 1.91, "train/epoch": 2.22, "train/global_step": 1000, "_timestamp": 1708939686.811165, "_runtime": 421.27890706062317, "_step": 3, "train/loss": 1.5765, "train/learning_rate": 0.0002777777777777778}
+{"eval/loss": 1.6712268590927124, "eval/bleu": 16.0438, "eval/gen_len": 11.7525, "eval/runtime": 26.0944, "eval/samples_per_second": 30.658, "eval/steps_per_second": 1.916, "train/epoch": 3.33, "train/global_step": 1500, "_timestamp": 1708939833.4617436, "_runtime": 567.9294855594635, "_step": 5, "train/loss": 1.2297, "train/learning_rate": 0.00016666666666666666}

wandb/run-20240226_092105-91uhs91a/logs/debug-internal.log

@@ -396,3 +396,77 @@
 2024-02-26 09:28:32,703 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
 2024-02-26 09:28:37,495 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
 2024-02-26 09:28:37,704 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:42,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:42,705 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:47,497 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:47,706 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:52,498 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:52,707 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:28:57,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:28:57,712 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:02,495 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:29:02,713 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:03,495 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:07,714 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:08,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:12,715 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:13,497 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:17,716 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:18,498 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:22,717 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:23,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:27,718 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:28,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:32,495 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:29:32,724 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:34,496 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:37,725 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:39,577 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/output.log
+2024-02-26 09:29:39,706 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:42,726 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:44,707 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:47,728 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:49,708 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:52,729 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:54,708 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:29:57,750 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:29:59,709 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:02,496 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:30:02,778 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:02,779 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-26 09:30:02,780 DEBUG SenderThread:196 [sender.py:send():382] send: history
+2024-02-26 09:30:02,781 DEBUG SenderThread:196 [sender.py:send_request():409] send_request: summary_record
+2024-02-26 09:30:02,781 INFO SenderThread:196 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-02-26 09:30:03,587 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/wandb-summary.json
+2024-02-26 09:30:04,782 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:07,779 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:09,783 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:12,781 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:14,783 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:17,782 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:19,784 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:22,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:24,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:27,785 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:29,786 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:32,497 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:30:32,787 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:33,462 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-26 09:30:33,463 DEBUG SenderThread:196 [sender.py:send():382] send: history
+2024-02-26 09:30:33,463 DEBUG SenderThread:196 [sender.py:send_request():409] send_request: summary_record
+2024-02-26 09:30:33,464 INFO SenderThread:196 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-02-26 09:30:33,596 INFO Thread-12 :196 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240226_092105-91uhs91a/files/wandb-summary.json
+2024-02-26 09:30:35,464 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:38,367 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:40,465 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:43,368 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:45,466 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:48,369 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:50,467 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:53,370 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:30:55,470 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:30:58,371 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:31:00,471 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report
+2024-02-26 09:31:02,498 DEBUG SenderThread:196 [sender.py:send():382] send: stats
+2024-02-26 09:31:03,372 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-26 09:31:05,499 DEBUG HandlerThread:196 [handler.py:handle_request():146] handle_request: status_report