Initial commit

.gitattributes

@@ -0,0 +1,2 @@
+data/**/* filter=lfs diff=lfs merge=lfs -text
+models/**/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

.python-version

@@ -0,0 +1 @@
+3.12

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e56bb3dc9a357e233a9fb7f8e4e8d78cf632912ee7cdb82ac7309ae13a444d
+size 1841

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-test.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6100e0a2b7a8da10e1826327f03572f062c8692094e6f79c80a9a0fea0ffe3c2
+size 283680

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-train-00000-of-00005.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02602e4416333875f19cc121f04e3f59d13fe6281fe9172684dd326cb63d90e9
+size 501967576

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-train-00001-of-00005.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96c471c057f51439eed145f3adb76a0199a769e0d04b09d1c307002b64631d04
+size 502355600

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-train-00002-of-00005.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6976fd4b3ea2373fb4db96de99459a0afefa4eaa2a60df6e04ce84f9ddb37d4f
+size 502181200

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-train-00003-of-00005.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f52c13815176e107801de98c486340647c0cfdee24d0384cc1b52bc29a32a1d
+size 501469240

data/AI-MO___numina_math-co_t/default/0.0.0/9d8d210c9f6a36c8f3cd84045668c9b7800ef517/numina_math-co_t-train-00004-of-00005.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:019ca18217d6570a600ee3c3fec9789ea6480dae2bb247c38ae89032b27245c1
+size 487732184

data/test.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe68549af30f20ef71cb8e885305e80ab40f38ce960a060b301d2f2df1bb9184
+size 140685

data/test_dataset/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77549e0a8318c2ad7f46f1c3df79f9a5e4bd3b2812a80a7f2b403bd0ea912cb
+size 1125192

data/test_dataset/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee8e05b4381db6d6cc7427c83f0536c1da684169b0b57e251a76d23a3d084670
+size 431

data/test_dataset/state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb96174acb2f77d300ea1fdc971d183ba564d79853319a2ec41c19615af41301
+size 298

data/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1d5690c7a443730e822249ea2e0b37799c330851f6c87342c0dcd5d77fc1126
+size 1243204995

data/train_dataset/data-00000-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b06f1e008b17e8ee80546673206e4ade8842dc0b840f3352db50d09d90c7072
+size 473206928

data/train_dataset/data-00001-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5036abbfeaf78a0b696fc26833d80c2b44ca32bd57f0a62eb1760ebf58261485
+size 473246368

data/train_dataset/data-00002-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45186315c884dbb0b03aefc8d7ebc4a43fe416cb1991acdc6643f8038806aea1
+size 472882424

data/train_dataset/data-00003-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4489cea42c1478171f9ed986fae4da66833daa184353b6d2638cd463d9e0d6e
+size 473004376

data/train_dataset/data-00004-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:572836a964eef213e9570d19b5abf74d8b9d99591e25e498d9f4141a8a845b4b
+size 473022936

data/train_dataset/data-00005-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a3d18f5c5a879daa106b1b596493471ca89075023d4a9aa3335c7aa866e7d06
+size 472883784

data/train_dataset/data-00006-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e38a00712bcb23bdd1a93a1c5299969607a25319542b3c24a0d0e26eadc36d1
+size 472906568

data/train_dataset/data-00007-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e6366921b4e83156591b110b91a85629cce7a56a2fc16922603f29da1b7e2da
+size 473162936

data/train_dataset/data-00008-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a99404147a03ed183bf5dad5c834de9a83bcd2dc2dc6531c0b8ad31fe5a113f1
+size 473061832

data/train_dataset/data-00009-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9593e6cfc36949e9107db0b17b06ad1f183cd9d3e0bb8afdc98b8f64bcf5765
+size 472899088

data/train_dataset/data-00010-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71bb48eed735d31e8c1e4e3503140fd3c7137c5aec0cec4dca2eb2ac12faf59d
+size 473032744

data/train_dataset/data-00011-of-00012.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04884fad18f5ab356bebce10fbf996407f78f432d12d14e693cf18fcdcb7f700
+size 472870416

data/train_dataset/dataset_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee8e05b4381db6d6cc7427c83f0536c1da684169b0b57e251a76d23a3d084670
+size 431

data/train_dataset/state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cfacc7784b691edb44b06521c361be2d888ed9cb961eb70584bab9303760293
+size 947

git-guide.txt

@@ -0,0 +1,28 @@
+xdg-open ~/.git-credentials
+xdg-open ~/.gitconfig
+xdg-open ~/.cache/huggingface
+
+huggingface-cli login
+huggingface-cli repo create "repo-name"
+
+git config --global user.email "[email protected]"
+git config --global user.name "username"
+
+uv init
+* git lfs track отправляет большие файлы в специальное хранилище; пушатся не сами файлы, а ссылки на них в хранилище
+git lfs track "data/**/*" (директория data; ** - все под-директории; * - все файлы)
+git lfs track "models/**/*"
+
+git add .gitattributes
+git add . (добавь все файлы, что находится в этой директории)
+
+git commit -m "Initial commit"
+
+git add git-guide.txt 
+git commit --amend --no-edit (если изменил какой-то файл после того, как объявил комит и ты не хочешь создавать новый)
+
+git remote add origin https://huggingface.co/Sierra-W117/math-gpt2
+(git remote remove origin)
+
+git branch -M main
+git push -u origin main

models/math-gpt2-v0/checkpoint-64/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:285d59ee8e224c185ad2239766d59b8c8c60fc4911b1fc98000ed47c06749151
+size 907

models/math-gpt2-v0/checkpoint-64/generation_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c831d77f4ccb7e22873162a06bdb0b846a588f6586890c6aab7cec1b7ef9a60e
+size 119

models/math-gpt2-v0/checkpoint-64/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12b69d3d4944bac6ac070c5912020130f1a85aac5557f469a55e3170f7ab338a
+size 497774208

models/math-gpt2-v0/checkpoint-64/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03bc453c349d0eb2faba453ac1ab28097404a05d27a942294b8269e8894f1a6
+size 995642298

models/math-gpt2-v0/checkpoint-64/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be2f08e0a34d93e3742746f4ecb1a5a6653f3b1da6db4b41f5f39cf3ae48a6f7
+size 14244

models/math-gpt2-v0/checkpoint-64/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d0278b77bd0a6a0013d1641e2fe9dd8de3641b1d4a7ec8656f671cec7965afc
+size 1064

models/math-gpt2-v0/checkpoint-64/trainer_state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf0bf27b94c3ebb86dcdf29d67753a85cd2510fcfbb14ef590bc7046bbe49eff
+size 744

models/math-gpt2-v0/checkpoint-64/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb4c73e15e28f4fa556da7bde75ca259de5e13418df731c6ac6b42092336fde5
+size 5304

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "math-gpt2"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "accelerate>=1.2.1",
+    "datasets>=3.2.0",
+    "ipykernel>=6.29.5",
+    "ipywidgets>=8.1.5",
+    "numpy>=2.2.1",
+    "torch>=2.5.1",
+    "transformers>=4.47.1",
+]

training.ipynb

@@ -0,0 +1,482 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "torch.cuda.is_available()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from datasets import load_dataset, Dataset, concatenate_datasets\n",
+    "from transformers import GPT2Tokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments, GPT2LMHeadModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['source', 'problem', 'solution', 'messages'],\n",
+       "        num_rows: 859494\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['source', 'problem', 'solution', 'messages'],\n",
+       "        num_rows: 100\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cache_dir = os.path.join(os.getcwd(), \"data\")\n",
+    "\n",
+    "dataset = load_dataset(\"AI-MO/NuminaMath-CoT\", cache_dir = cache_dir)\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Problem: Find all solutions to the equation $\\displaystyle\\sqrt[3]{3 - \\frac{x}{3}} = -2$. \n",
+      "Solution: Start by isolating the cube root:\n",
+      "$$ \\sqrt[3]{3 - \\frac{x}{3}} = -2 $$\n",
+      "\n",
+      "Cube both sides to eliminate the cube root:\n",
+      "$$ 3 - \\frac{x}{3} = (-2)^3 $$\n",
+      "$$ 3 - \\frac{x}{3} = -8 $$\n",
+      "\n",
+      "Solve for $x$:\n",
+      "$$ 3 + 8 = \\frac{x}{3} $$\n",
+      "$$ 11 = \\frac{x}{3} $$\n",
+      "$$ x = 33 $$\n",
+      "\n",
+      "Thus, the solution to the equation is:\n",
+      "$$ \\boxed{x = 33} $$\n"
+     ]
+    }
+   ],
+   "source": [
+    "constanta = 4\n",
+    "\n",
+    "problem = dataset['train']['problem'][constanta]\n",
+    "solution = dataset['train']['solution'][constanta]\n",
+    "\n",
+    "print(f\"Problem: {problem} \\nSolution: {solution}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_text(item):\n",
+    "    return f\"Problem: {item['problem']}\\nSolution: {item['solution']}\\n\\n\"\n",
+    "\n",
+    "def prepare_dataset(dataset_dict, file_name, cache_dir):\n",
+    "\n",
+    "    file = os.path.join(cache_dir, f\"{file_name}.txt\")\n",
+    "\n",
+    "    if os.path.exists(file):\n",
+    "        return file\n",
+    "    \n",
+    "    with open(file, 'w', encoding='utf-8') as f:\n",
+    "        for item in dataset_dict:\n",
+    "            f.write(format_text(item))\n",
+    "    \n",
+    "    return file\n",
+    "\n",
+    "train_file = prepare_dataset(\n",
+    "    dataset_dict = dataset['train'],\n",
+    "    file_name = \"train\",\n",
+    "    cache_dir = cache_dir\n",
+    ")\n",
+    "\n",
+    "test_file = prepare_dataset(\n",
+    "    dataset_dict = dataset['test'],\n",
+    "    file_name = \"test\",\n",
+    "    cache_dir = cache_dir\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "26997273"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with open(train_file, 'r') as f:\n",
+    "    num_lines = sum(1 for line in f)\n",
+    "num_lines"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tokenize_function(examples, tokenizer):\n",
+    "    item = tokenizer(\n",
+    "        examples['text'],\n",
+    "        truncation = True,\n",
+    "        max_length = 64,\n",
+    "        padding = True,\n",
+    "        return_tensors = 'pt'\n",
+    "    ).to(\"cuda\")\n",
+    "    return item\n",
+    "\n",
+    "\n",
+    "def process_chunk(text_chunk, tokenizer):\n",
+    "    dataset = Dataset.from_dict({\n",
+    "        'text': text_chunk\n",
+    "    }).map(\n",
+    "        lambda x: tokenize_function(x, tokenizer),\n",
+    "        batched = True\n",
+    "    )\n",
+    "    return dataset\n",
+    "\n",
+    "\n",
+    "def create_dataset(file, tokenizer, dataset_name, cache_dir, chunk_size = 750_000, max_chunks = 20):\n",
+    "    dataset_path = os.path.join(cache_dir, dataset_name)\n",
+    "\n",
+    "    if os.path.exists(dataset_path):\n",
+    "        return Dataset.load_from_disk(dataset_path)\n",
+    "    \n",
+    "    datasets = []\n",
+    "    chunks_processed = 0\n",
+    "    \n",
+    "    with open(file, 'r', encoding = 'utf-8') as f:\n",
+    "        while chunks_processed < max_chunks:\n",
+    "            text_chunk = []\n",
+    "            while len(text_chunk) < chunk_size:\n",
+    "                line = f.readline()\n",
+    "                if not line:\n",
+    "                    break\n",
+    "                text_chunk.append(line)\n",
+    "            \n",
+    "            if text_chunk:\n",
+    "                datasets.append(process_chunk(text_chunk, tokenizer))\n",
+    "                chunks_processed += 1\n",
+    "    \n",
+    "\n",
+    "    the_dataset = concatenate_datasets(datasets)\n",
+    "    the_dataset.save_to_disk(dataset_path)\n",
+    "    \n",
+    "    return the_dataset\n",
+    "\n",
+    "\n",
+    "train_dataset = create_dataset(\n",
+    "    file = train_file, \n",
+    "    tokenizer = tokenizer,\n",
+    "    dataset_name = \"train_dataset\",\n",
+    "    cache_dir = cache_dir\n",
+    "    )\n",
+    "\n",
+    "test_dataset = create_dataset(\n",
+    "    file = test_file, \n",
+    "    tokenizer = tokenizer,\n",
+    "    dataset_name = \"test_dataset\",\n",
+    "    cache_dir = cache_dir,\n",
+    "    max_chunks = 1\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_collator = DataCollatorForLanguageModeling(\n",
+    "    tokenizer = tokenizer,\n",
+    "    mlm = False\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'text': 'Problem: Suppose that $g(x) = 5x - 3$. What is $g^{-1}(g^{-1}(14))$?\\n', 'input_ids': [40781, 25, 39200, 326, 720, 70, 7, 87, 8, 796, 642, 87, 532, 513, 35307, 1867, 318, 720, 70, 36796, 12, 16, 92, 7, 70, 36796, 12, 16, 92, 7, 1415, 4008, 3, 30, 198, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(train_dataset[13])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Problem: Suppose that $g(x) = 5x - 3$. What is $g^{-1}(g^{-1}(14))$?\n",
+      "<|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|>\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(train_dataset[13]['input_ids']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compute_metrics(eval_pred):\n",
+    "    predictions, labels = eval_pred\n",
+    "    \n",
+    "    loss = torch.nn.CrossEntropyLoss()(\n",
+    "        torch.tensor(predictions).cuda(), \n",
+    "        torch.tensor(labels).cuda()\n",
+    "    ).item()\n",
+    "    \n",
+    "    return loss"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "15000000"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(train_dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_dataset = train_dataset.select(range(len(train_dataset)//15_000))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.makedirs(\"models\", exist_ok = True)\n",
+    "os.makedirs(\"models/math-gpt2-v0\", exist_ok = True)\n",
+    "os.makedirs(\"models/math-gpt2-v0/logs\", exist_ok = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gc\n",
+    "\n",
+    "gc.collect()\n",
+    "torch.cuda.empty_cache()\n",
+    "\n",
+    "# идея - сохранить лучший и оба по обе стороны\n",
+    "# правильно ли я понимаю, что в текущих условиях, будет соохранена только одна модель, а logs - все"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
+    "model.resize_token_embeddings(len(tokenizer))\n",
+    "model = model.to(\"cuda\")\n",
+    "\n",
+    "training_args = TrainingArguments(\n",
+    "\n",
+    "    output_dir = \"models/math-gpt2-v0\",             # директория для сохранения модели\n",
+    "    logging_dir = \"models/math-gpt2-v0/logs\",       # куда будут сохраняться логи обучения в формате TensorBoard \n",
+    "    run_name = \"math-gpt2-v0\",                      # название эксперимента в системах логирования (например, TensorBoard)\n",
+    "    overwrite_output_dir = False,\n",
+    "    save_total_limit = 1, # сохранить модель с лучшим счетом из всех steps\n",
+    "    save_strategy = \"steps\",\n",
+    "    \n",
+    "    do_eval = True,                     # валидация\n",
+    "    eval_strategy = \"steps\",            # стратегия запуска волидации: по эпохам или по шагам\n",
+    "    eval_steps = 10000,                   # количество шагов между валидациями\n",
+    "\n",
+    "    load_best_model_at_end = True,\n",
+    "    metric_for_best_model = \"cross_entropy\",    # по какой метрике оцениваем\n",
+    "    greater_is_better = False,                  # хотим ее минимизировать или максимизировать\n",
+    "\n",
+    "    num_train_epochs = 4,\n",
+    "    per_device_train_batch_size = 32,   # количество примеров для обучения на одном устройстве GPU\n",
+    "    per_device_eval_batch_size = 64, \n",
+    "    gradient_checkpointing = True,      # экономит память GPU за счет пересчета некоторых промежуточных активаций вместо их хранения\n",
+    "    fp16 = True,                        # использование 16-битных чисел с плавающей точкой вместо стандартных 32-битных (fp32)\n",
+    "\n",
+    "    gradient_accumulation_steps = 2,    # накапливает градиенты N батчей перед обновлением весов\n",
+    "    eval_accumulation_steps = 2,        # накапливает результаты N батчей при валидации\n",
+    "\n",
+    "    learning_rate = 5e-5,               # ОТВЕТ ЗНАТОКА: это базовый lr\n",
+    "    warmup_steps = 500,                 # Количество шагов для прогрева (warmup) learning rate. Прогрев в этом случае - обновление learning rate.\n",
+    "    weight_decay = 0.01,                # ниже тоже базовые, знаю\n",
+    "    adam_beta1 = 0.9,\n",
+    "    adam_beta2 = 0.999,\n",
+    "    max_grad_norm = 1.0,\n",
+    "    \n",
+    "    save_steps = 10000,                   # частота сохранения модели\n",
+    "    logging_steps = 10000,                # частота логирования метрик\n",
+    "    push_to_hub = False,                # загрузка модели на HuggingFace\n",
+    ")\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model = model,\n",
+    "    args = training_args,\n",
+    "    data_collator = data_collator,\n",
+    "    train_dataset = train_dataset,\n",
+    "    eval_dataset = test_dataset,\n",
+    "    compute_metrics = compute_metrics\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c4125cf1faae456796edb8f452529acb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/64 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'train_runtime': 27.7282, 'train_samples_per_second': 144.257, 'train_steps_per_second': 2.308, 'train_loss': 6.928213119506836, 'epoch': 4.0}\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=64, training_loss=6.928213119506836, metrics={'train_runtime': 27.7282, 'train_samples_per_second': 144.257, 'train_steps_per_second': 2.308, 'total_flos': 130646016000000.0, 'train_loss': 6.928213119506836, 'epoch': 4.0})"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

using.ipynb

@@ -0,0 +1,156 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "aebdeaaf529549a1ad39ccf838e39e71",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "26da6488303344ce90900207ae33b6bd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1eea77d125a74e00b1f050960dfa42aa",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b704ae3896dc46698ca50b7a29230bde",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "42eaaf035fad40f583ee01dac1a86e85",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
+    "\n",
+    "model = GPT2LMHeadModel.from_pretrained(\"models/math-gpt2-v0/checkpoint-64\")\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
+    "model = model.to(\"cuda\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Your math prompt here:\n",
+      "\n",
+      "$ x = (m^3 + k) + (m^4 + k) + (m^5 + k)\n",
+      "\n",
+      "Now, we have:\n",
+      "\n",
+      "$ x = (m^3 +\n"
+     ]
+    }
+   ],
+   "source": [
+    "def generate_text(prompt, max_length=50):\n",
+    "\n",
+    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\n",
+    "    \n",
+    "    outputs = model.generate(\n",
+    "        inputs[\"input_ids\"],\n",
+    "        max_length=max_length,\n",
+    "        pad_token_id=tokenizer.eos_token_id,\n",
+    "        num_return_sequences=1,\n",
+    "        do_sample=True,\n",
+    "        temperature=0.7\n",
+    "    )\n",
+    "    \n",
+    "    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
+    "\n",
+    "prompt = \"Your math prompt here\"\n",
+    "generated_text = generate_text(prompt)\n",
+    "print(generated_text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}