Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-2695/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,45 @@
+---
+tags:
+- autotrain
+- text-generation-inference
+- text-generation
+- peft
+library_name: transformers
+widget:
+ - messages:
+ - role: user
+ content: What is your favorite condiment?
+license: other
+---
+
+# Model Trained Using AutoTrain
+
+This model was trained using AutoTrain. For more information, please visit [AutoTrain](https://hf.co/docs/autotrain).
+
+# Usage
+
+```python
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_path = "PATH_TO_THIS_REPO"
+
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForCausalLM.from_pretrained(
+ model_path,
+ device_map="auto",
+ torch_dtype='auto'
+).eval()
+
+# Prompt content: "hi"
+messages = [
+ {"role": "user", "content": "hi"}
+]
+
+input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, add_generation_prompt=True, return_tensors='pt')
+output_ids = model.generate(input_ids.to('cuda'))
+response = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)
+
+# Model response: "Hello! How can I assist you today?"
+print(response)
+```

adapter_config.json

@@ -0,0 +1,34 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-2b-it",
+ "bias": "none",
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "q_proj",
+ "up_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "gate_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c867772ff6302167afed5fade5d54e7c6278fa1b04f48d0c5d759a00bff1cb7
+size 78480072

autotrain-data/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "validation"]}

autotrain-data/train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9d39aad035965d8d890e340584f91728655e1d09e4e930dbe96f0e3d0b8cd5
+size 99839504

autotrain-data/train/dataset_info.json

@@ -0,0 +1,16 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "response": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "autotrain_text": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}

autotrain-data/train/state.json

@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "8bd7c0e2adb74fd0",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}

autotrain-data/validation/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9d39aad035965d8d890e340584f91728655e1d09e4e930dbe96f0e3d0b8cd5
+size 99839504

autotrain-data/validation/dataset_info.json

@@ -0,0 +1,16 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "response": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "autotrain_text": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}

autotrain-data/validation/state.json

@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "8bd7c0e2adb74fd0",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}

checkpoint-2695/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google/gemma-2b-it
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.10.0

checkpoint-2695/adapter_config.json

@@ -0,0 +1,34 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-2b-it",
+ "bias": "none",
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "q_proj",
+ "up_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "gate_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}

checkpoint-2695/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c867772ff6302167afed5fade5d54e7c6278fa1b04f48d0c5d759a00bff1cb7
+size 78480072

checkpoint-2695/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09e47b3aa5ff5d1ee900147dc936d3375c819c606e6afdaaa20a03bcbbf34a89
+size 157104826

checkpoint-2695/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d661518168fd4f63ccc7f86efee8168b53fe82dea4c6d5a3a94430425708002
+size 888

checkpoint-2695/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48dd287b33de6260c6d8d34590b36e79d271dae3a783d52c20d4730c559a8c02
+size 14244

checkpoint-2695/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1a254b98a24661fb0e693ef795e4c9a7485063be60fa5d404d1fdbd683ba239
+size 1064

checkpoint-2695/special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+ "additional_special_tokens": [
+ "<start_of_turn>",
+ "<end_of_turn>"
+ ],
+ "bos_token": {
+ "content": "<bos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "<pad>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "unk_token": {
+ "content": "<unk>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}

checkpoint-2695/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e97791a5e007260de1db7e1692e53150e08cea481e2bf25435553380c147ee
+size 17477929

checkpoint-2695/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

checkpoint-2695/tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "added_tokens_decoder": {
+ "0": {
+ "content": "<pad>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "<bos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "3": {
+ "content": "<unk>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "106": {
+ "content": "<start_of_turn>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "107": {
+ "content": "<end_of_turn>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [
+ "<start_of_turn>",
+ "<end_of_turn>"
+ ],
+ "bos_token": "<bos>",
+ "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<eos>",
+ "legacy": null,
+ "model_max_length": 2048,
+ "pad_token": "<pad>",
+ "sp_model_kwargs": {},
+ "spaces_between_special_tokens": false,
+ "tokenizer_class": "GemmaTokenizer",
+ "unk_token": "<unk>",
+ "use_default_system_prompt": false
+}

checkpoint-2695/trainer_state.json

@@ -0,0 +1,770 @@
+{
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 0.9997217842900863,
+ "eval_steps": 500,
+ "global_step": 2695,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.01,
+ "grad_norm": 25.05885124206543,
+ "learning_rate": 2.666666666666667e-06,
+ "loss": 3.8791,
+ "step": 25
+ },
+ {
+ "epoch": 0.02,
+ "grad_norm": 5.391462326049805,
+ "learning_rate": 5.333333333333334e-06,
+ "loss": 2.5036,
+ "step": 50
+ },
+ {
+ "epoch": 0.03,
+ "grad_norm": 9.238971710205078,
+ "learning_rate": 8.111111111111112e-06,
+ "loss": 1.7464,
+ "step": 75
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 13.056879043579102,
+ "learning_rate": 1.088888888888889e-05,
+ "loss": 1.2491,
+ "step": 100
+ },
+ {
+ "epoch": 0.05,
+ "grad_norm": 4.893646717071533,
+ "learning_rate": 1.3666666666666667e-05,
+ "loss": 0.4782,
+ "step": 125
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 0.030804015696048737,
+ "learning_rate": 1.6444444444444444e-05,
+ "loss": 0.0186,
+ "step": 150
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 0.11937689036130905,
+ "learning_rate": 1.922222222222222e-05,
+ "loss": 0.002,
+ "step": 175
+ },
+ {
+ "epoch": 0.07,
+ "grad_norm": 0.0017061774851754308,
+ "learning_rate": 2.2e-05,
+ "loss": 0.0001,
+ "step": 200
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.0143441678956151,
+ "learning_rate": 2.477777777777778e-05,
+ "loss": 0.0,
+ "step": 225
+ },
+ {
+ "epoch": 0.09,
+ "grad_norm": 0.001103504910133779,
+ "learning_rate": 2.7555555555555555e-05,
+ "loss": 0.0,
+ "step": 250
+ },
+ {
+ "epoch": 0.1,
+ "grad_norm": 0.001787293003872037,
+ "learning_rate": 2.9962886597938144e-05,
+ "loss": 0.0001,
+ "step": 275
+ },
+ {
+ "epoch": 0.11,
+ "grad_norm": 0.0007679874543100595,
+ "learning_rate": 2.965360824742268e-05,
+ "loss": 0.0,
+ "step": 300
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 0.0011675716377794743,
+ "learning_rate": 2.934432989690722e-05,
+ "loss": 0.0,
+ "step": 325
+ },
+ {
+ "epoch": 0.13,
+ "grad_norm": 0.0006628546980209649,
+ "learning_rate": 2.903505154639175e-05,
+ "loss": 0.0,
+ "step": 350
+ },
+ {
+ "epoch": 0.14,
+ "grad_norm": 0.0004541067173704505,
+ "learning_rate": 2.872577319587629e-05,
+ "loss": 0.0,
+ "step": 375
+ },
+ {
+ "epoch": 0.15,
+ "grad_norm": 0.0036788100842386484,
+ "learning_rate": 2.8416494845360828e-05,
+ "loss": 0.0,
+ "step": 400
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.003441563807427883,
+ "learning_rate": 2.810721649484536e-05,
+ "loss": 0.0,
+ "step": 425
+ },
+ {
+ "epoch": 0.17,
+ "grad_norm": 0.0021586541552096605,
+ "learning_rate": 2.77979381443299e-05,
+ "loss": 0.0,
+ "step": 450
+ },
+ {
+ "epoch": 0.18,
+ "grad_norm": 0.002755054971203208,
+ "learning_rate": 2.7488659793814434e-05,
+ "loss": 0.0,
+ "step": 475
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 0.001827051630243659,
+ "learning_rate": 2.717938144329897e-05,
+ "loss": 0.0,
+ "step": 500
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 0.01897139474749565,
+ "learning_rate": 2.6870103092783505e-05,
+ "loss": 0.0,
+ "step": 525
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 0.00023522872652392834,
+ "learning_rate": 2.6560824742268044e-05,
+ "loss": 0.0,
+ "step": 550
+ },
+ {
+ "epoch": 0.21,
+ "grad_norm": 0.00029744170024059713,
+ "learning_rate": 2.6251546391752576e-05,
+ "loss": 0.0,
+ "step": 575
+ },
+ {
+ "epoch": 0.22,
+ "grad_norm": 0.00024159494205377996,
+ "learning_rate": 2.5942268041237114e-05,
+ "loss": 0.0,
+ "step": 600
+ },
+ {
+ "epoch": 0.23,
+ "grad_norm": 0.0010275288950651884,
+ "learning_rate": 2.563298969072165e-05,
+ "loss": 0.0,
+ "step": 625
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.001225774991326034,
+ "learning_rate": 2.5323711340206185e-05,
+ "loss": 0.0,
+ "step": 650
+ },
+ {
+ "epoch": 0.25,
+ "grad_norm": 0.00023019347281660885,
+ "learning_rate": 2.501443298969072e-05,
+ "loss": 0.0,
+ "step": 675
+ },
+ {
+ "epoch": 0.26,
+ "grad_norm": 0.0008600191795267165,
+ "learning_rate": 2.470515463917526e-05,
+ "loss": 0.0,
+ "step": 700
+ },
+ {
+ "epoch": 0.27,
+ "grad_norm": 0.00018496481061447412,
+ "learning_rate": 2.4395876288659795e-05,
+ "loss": 0.0,
+ "step": 725
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.0008189578074961901,
+ "learning_rate": 2.408659793814433e-05,
+ "loss": 0.0,
+ "step": 750
+ },
+ {
+ "epoch": 0.29,
+ "grad_norm": 0.0006728855660185218,
+ "learning_rate": 2.377731958762887e-05,
+ "loss": 0.0,
+ "step": 775
+ },
+ {
+ "epoch": 0.3,
+ "grad_norm": 0.00020031195890624076,
+ "learning_rate": 2.34680412371134e-05,
+ "loss": 0.0,
+ "step": 800
+ },
+ {
+ "epoch": 0.31,
+ "grad_norm": 0.0011467556469142437,
+ "learning_rate": 2.315876288659794e-05,
+ "loss": 0.0,
+ "step": 825
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.0001870268170023337,
+ "learning_rate": 2.2849484536082475e-05,
+ "loss": 0.0,
+ "step": 850
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.0006849793717265129,
+ "learning_rate": 2.254020618556701e-05,
+ "loss": 0.0,
+ "step": 875
+ },
+ {
+ "epoch": 0.33,
+ "grad_norm": 0.000808037118986249,
+ "learning_rate": 2.2230927835051546e-05,
+ "loss": 0.0,
+ "step": 900
+ },
+ {
+ "epoch": 0.34,
+ "grad_norm": 0.0001667607866693288,
+ "learning_rate": 2.1921649484536084e-05,
+ "loss": 0.0,
+ "step": 925
+ },
+ {
+ "epoch": 0.35,
+ "grad_norm": 0.0007605258724652231,
+ "learning_rate": 2.1612371134020616e-05,
+ "loss": 0.0,
+ "step": 950
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.00016520808276254684,
+ "learning_rate": 2.1303092783505155e-05,
+ "loss": 0.0,
+ "step": 975
+ },
+ {
+ "epoch": 0.37,
+ "grad_norm": 0.0001515819749329239,
+ "learning_rate": 2.099381443298969e-05,
+ "loss": 0.0,
+ "step": 1000
+ },
+ {
+ "epoch": 0.38,
+ "grad_norm": 0.010785164311528206,
+ "learning_rate": 2.0684536082474226e-05,
+ "loss": 0.0,
+ "step": 1025
+ },
+ {
+ "epoch": 0.39,
+ "grad_norm": 0.0008493285276927054,
+ "learning_rate": 2.037525773195876e-05,
+ "loss": 0.0,
+ "step": 1050
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.00024162570480257273,
+ "learning_rate": 2.00659793814433e-05,
+ "loss": 0.0,
+ "step": 1075
+ },
+ {
+ "epoch": 0.41,
+ "grad_norm": 0.00014333316357806325,
+ "learning_rate": 1.9756701030927835e-05,
+ "loss": 0.0,
+ "step": 1100
+ },
+ {
+ "epoch": 0.42,
+ "grad_norm": 0.00013407158257905394,
+ "learning_rate": 1.944742268041237e-05,
+ "loss": 0.0,
+ "step": 1125
+ },
+ {
+ "epoch": 0.43,
+ "grad_norm": 0.00013438447786029428,
+ "learning_rate": 1.913814432989691e-05,
+ "loss": 0.0,
+ "step": 1150
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 0.0003521848120726645,
+ "learning_rate": 1.882886597938144e-05,
+ "loss": 0.0,
+ "step": 1175
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.0005705170915462077,
+ "learning_rate": 1.851958762886598e-05,
+ "loss": 0.0,
+ "step": 1200
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.0003134581202175468,
+ "learning_rate": 1.8210309278350516e-05,
+ "loss": 0.0,
+ "step": 1225
+ },
+ {
+ "epoch": 0.46,
+ "grad_norm": 0.0001157010265160352,
+ "learning_rate": 1.790103092783505e-05,
+ "loss": 0.0,
+ "step": 1250
+ },
+ {
+ "epoch": 0.47,
+ "grad_norm": 0.00011368603736627847,
+ "learning_rate": 1.7591752577319586e-05,
+ "loss": 0.0,
+ "step": 1275
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.00016138765204232186,
+ "learning_rate": 1.7282474226804125e-05,
+ "loss": 0.0,
+ "step": 1300
+ },
+ {
+ "epoch": 0.49,
+ "grad_norm": 0.00011523786088218912,
+ "learning_rate": 1.697319587628866e-05,
+ "loss": 0.0,
+ "step": 1325
+ },
+ {
+ "epoch": 0.5,
+ "grad_norm": 0.00016504651284776628,
+ "learning_rate": 1.6663917525773196e-05,
+ "loss": 0.0,
+ "step": 1350
+ },
+ {
+ "epoch": 0.51,
+ "grad_norm": 0.0002800218644551933,
+ "learning_rate": 1.635463917525773e-05,
+ "loss": 0.0,
+ "step": 1375
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 0.00010265475430060178,
+ "learning_rate": 1.604536082474227e-05,
+ "loss": 0.0,
+ "step": 1400
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.00560023495927453,
+ "learning_rate": 1.5736082474226806e-05,
+ "loss": 0.0,
+ "step": 1425
+ },
+ {
+ "epoch": 0.54,
+ "grad_norm": 0.00010008086974266917,
+ "learning_rate": 1.542680412371134e-05,
+ "loss": 0.0,
+ "step": 1450
+ },
+ {
+ "epoch": 0.55,
+ "grad_norm": 9.670985309639946e-05,
+ "learning_rate": 1.5117525773195878e-05,
+ "loss": 0.0,
+ "step": 1475
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 9.528211376164109e-05,
+ "learning_rate": 1.4808247422680412e-05,
+ "loss": 0.0,
+ "step": 1500
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.0001707332266960293,
+ "learning_rate": 1.4498969072164949e-05,
+ "loss": 0.0,
+ "step": 1525
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.00045562253217212856,
+ "learning_rate": 1.4189690721649484e-05,
+ "loss": 0.0,
+ "step": 1550
+ },
+ {
+ "epoch": 0.58,
+ "grad_norm": 9.548822708893567e-05,
+ "learning_rate": 1.388041237113402e-05,
+ "loss": 0.0,
+ "step": 1575
+ },
+ {
+ "epoch": 0.59,
+ "grad_norm": 0.00022835374693386257,
+ "learning_rate": 1.3571134020618558e-05,
+ "loss": 0.0,
+ "step": 1600
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 0.00044599550892598927,
+ "learning_rate": 1.3261855670103094e-05,
+ "loss": 0.0,
+ "step": 1625
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.0001324521581409499,
+ "learning_rate": 1.2952577319587629e-05,
+ "loss": 0.0,
+ "step": 1650
+ },
+ {
+ "epoch": 0.62,
+ "grad_norm": 7.690698112128302e-05,
+ "learning_rate": 1.2643298969072166e-05,
+ "loss": 0.0,
+ "step": 1675
+ },
+ {
+ "epoch": 0.63,
+ "grad_norm": 0.00026828458067029715,
+ "learning_rate": 1.2334020618556701e-05,
+ "loss": 0.0,
+ "step": 1700
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 7.547908899141476e-05,
+ "learning_rate": 1.2024742268041237e-05,
+ "loss": 0.0,
+ "step": 1725
+ },
+ {
+ "epoch": 0.65,
+ "grad_norm": 6.959487654967234e-05,
+ "learning_rate": 1.1715463917525774e-05,
+ "loss": 0.0,
+ "step": 1750
+ },
+ {
+ "epoch": 0.66,
+ "grad_norm": 0.0002681033220142126,
+ "learning_rate": 1.140618556701031e-05,
+ "loss": 0.0,
+ "step": 1775
+ },
+ {
+ "epoch": 0.67,
+ "grad_norm": 7.833562267478555e-05,
+ "learning_rate": 1.1096907216494845e-05,
+ "loss": 0.0,
+ "step": 1800
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 0.00029178851400502026,
+ "learning_rate": 1.0787628865979382e-05,
+ "loss": 0.0,
+ "step": 1825
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.0002143761230399832,
+ "learning_rate": 1.0478350515463917e-05,
+ "loss": 0.0,
+ "step": 1850
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.00024392092018388212,
+ "learning_rate": 1.0169072164948452e-05,
+ "loss": 0.0,
+ "step": 1875
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.005321436561644077,
+ "learning_rate": 9.85979381443299e-06,
+ "loss": 0.0,
+ "step": 1900
+ },
+ {
+ "epoch": 0.71,
+ "grad_norm": 0.00020479953673202544,
+ "learning_rate": 9.550515463917525e-06,
+ "loss": 0.0,
+ "step": 1925
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 9.459372813580558e-05,
+ "learning_rate": 9.241237113402064e-06,
+ "loss": 0.0,
+ "step": 1950
+ },
+ {
+ "epoch": 0.73,
+ "grad_norm": 6.89596199663356e-05,
+ "learning_rate": 8.931958762886599e-06,
+ "loss": 0.0,
+ "step": 1975
+ },
+ {
+ "epoch": 0.74,
+ "grad_norm": 0.00019562170200515538,
+ "learning_rate": 8.622680412371134e-06,
+ "loss": 0.0,
+ "step": 2000
+ },
+ {
+ "epoch": 0.75,
+ "grad_norm": 5.944158328929916e-05,
+ "learning_rate": 8.313402061855671e-06,
+ "loss": 0.0,
+ "step": 2025
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 7.170682511059567e-05,
+ "learning_rate": 8.004123711340207e-06,
+ "loss": 0.0,
+ "step": 2050
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.00020228260837029666,
+ "learning_rate": 7.694845360824742e-06,
+ "loss": 0.0,
+ "step": 2075
+ },
+ {
+ "epoch": 0.78,
+ "grad_norm": 0.0001803228515200317,
+ "learning_rate": 7.3855670103092784e-06,
+ "loss": 0.0,
+ "step": 2100
+ },
+ {
+ "epoch": 0.79,
+ "grad_norm": 7.207765156636015e-05,
+ "learning_rate": 7.076288659793815e-06,
+ "loss": 0.0,
+ "step": 2125
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.00012546585639938712,
+ "learning_rate": 6.76701030927835e-06,
+ "loss": 0.0,
+ "step": 2150
+ },
+ {
+ "epoch": 0.81,
+ "grad_norm": 7.152390026021749e-05,
+ "learning_rate": 6.457731958762886e-06,
+ "loss": 0.0,
+ "step": 2175
+ },
+ {
+ "epoch": 0.82,
+ "grad_norm": 0.0001733840472297743,
+ "learning_rate": 6.148453608247423e-06,
+ "loss": 0.0,
+ "step": 2200
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.0002956937823910266,
+ "learning_rate": 5.8391752577319596e-06,
+ "loss": 0.0,
+ "step": 2225
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.000241271874983795,
+ "learning_rate": 5.529896907216495e-06,
+ "loss": 0.0,
+ "step": 2250
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 7.127185381250456e-05,
+ "learning_rate": 5.220618556701031e-06,
+ "loss": 0.0,
+ "step": 2275
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.00015488360077142715,
+ "learning_rate": 4.911340206185567e-06,
+ "loss": 0.0,
+ "step": 2300
+ },
+ {
+ "epoch": 0.86,
+ "grad_norm": 7.13276385795325e-05,
+ "learning_rate": 4.602061855670103e-06,
+ "loss": 0.0,
+ "step": 2325
+ },
+ {
+ "epoch": 0.87,
+ "grad_norm": 7.412026025122032e-05,
+ "learning_rate": 4.292783505154639e-06,
+ "loss": 0.0,
+ "step": 2350
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.00010627138544805348,
+ "learning_rate": 3.983505154639176e-06,
+ "loss": 0.0,
+ "step": 2375
+ },
+ {
+ "epoch": 0.89,
+ "grad_norm": 6.658281927229837e-05,
+ "learning_rate": 3.6742268041237114e-06,
+ "loss": 0.0,
+ "step": 2400
+ },
+ {
+ "epoch": 0.9,
+ "grad_norm": 6.946924258954823e-05,
+ "learning_rate": 3.3649484536082477e-06,
+ "loss": 0.0,
+ "step": 2425
+ },
+ {
+ "epoch": 0.91,
+ "grad_norm": 9.910303197102621e-05,
+ "learning_rate": 3.0556701030927835e-06,
+ "loss": 0.0,
+ "step": 2450
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 6.903583562234417e-05,
+ "learning_rate": 2.7463917525773197e-06,
+ "loss": 0.0,
+ "step": 2475
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.0002056838129647076,
+ "learning_rate": 2.437113402061856e-06,
+ "loss": 0.0,
+ "step": 2500
+ },
+ {
+ "epoch": 0.94,
+ "grad_norm": 0.00023039935331325978,
+ "learning_rate": 2.1278350515463917e-06,
+ "loss": 0.0,
+ "step": 2525
+ },
+ {
+ "epoch": 0.95,
+ "grad_norm": 7.011190609773621e-05,
+ "learning_rate": 1.8185567010309277e-06,
+ "loss": 0.0,
+ "step": 2550
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 7.29473540559411e-05,
+ "learning_rate": 1.509278350515464e-06,
+ "loss": 0.0,
+ "step": 2575
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 9.540535393171012e-05,
+ "learning_rate": 1.2000000000000002e-06,
+ "loss": 0.0,
+ "step": 2600
+ },
+ {
+ "epoch": 0.97,
+ "grad_norm": 7.17702423571609e-05,
+ "learning_rate": 8.907216494845361e-07,
+ "loss": 0.0,
+ "step": 2625
+ },
+ {
+ "epoch": 0.98,
+ "grad_norm": 6.912889512022957e-05,
+ "learning_rate": 5.814432989690722e-07,
+ "loss": 0.0,
+ "step": 2650
+ },
+ {
+ "epoch": 0.99,
+ "grad_norm": 7.510351133532822e-05,
+ "learning_rate": 2.7216494845360824e-07,
+ "loss": 0.0,
+ "step": 2675
+ }
+ ],
+ "logging_steps": 25,
+ "max_steps": 2695,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 1,
+ "save_steps": 500,
+ "total_flos": 1.3256372778958848e+17,
+ "train_batch_size": 1,
+ "trial_name": null,
+ "trial_params": null
+}

checkpoint-2695/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28d2e78c2bd95b653ee66408d827479fbb68511b78e9cbf4d959ef34ec8d07e1
+size 4984

runs/Mar27_11-14-19_DESKTOP-09EJKS6/events.out.tfevents.1711505690.DESKTOP-09EJKS6.16672.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b55689db9a7dac452d9e9c39ba16eccde459154f00665ab2ffdbcce7a48144a8
-size 27559
+oid sha256:8e7ec56e53d2ca8288933e57c92ed6be8f5ed06be1a9b2746383d8cbcf31f79a
+size 28124

special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+ "additional_special_tokens": [
+ "<start_of_turn>",
+ "<end_of_turn>"
+ ],
+ "bos_token": {
+ "content": "<bos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "<pad>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "unk_token": {
+ "content": "<unk>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e97791a5e007260de1db7e1692e53150e08cea481e2bf25435553380c147ee
+size 17477929

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "added_tokens_decoder": {
+ "0": {
+ "content": "<pad>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "<bos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "3": {
+ "content": "<unk>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "106": {
+ "content": "<start_of_turn>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "107": {
+ "content": "<end_of_turn>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [
+ "<start_of_turn>",
+ "<end_of_turn>"
+ ],
+ "bos_token": "<bos>",
+ "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<eos>",
+ "legacy": null,
+ "model_max_length": 2048,
+ "pad_token": "<pad>",
+ "sp_model_kwargs": {},
+ "spaces_between_special_tokens": false,
+ "tokenizer_class": "GemmaTokenizer",
+ "unk_token": "<unk>",
+ "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28d2e78c2bd95b653ee66408d827479fbb68511b78e9cbf4d959ef34ec8d07e1
+size 4984

training_params.json

@@ -0,0 +1,47 @@
+{
+ "model": "google/gemma-2b-it",
+ "project_name": "gemma-2b-it-adress-bot-seoul-epochs-1",
+ "data_path": "gemma-2b-it-adress-bot-seoul-epochs-1/autotrain-data",
+ "train_split": "train",
+ "valid_split": null,
+ "add_eos_token": true,
+ "block_size": 1024,
+ "model_max_length": 2048,
+ "padding": "right",
+ "trainer": "sft",
+ "use_flash_attention_2": false,
+ "log": "tensorboard",
+ "disable_gradient_checkpointing": false,
+ "logging_steps": -1,
+ "evaluation_strategy": "epoch",
+ "save_total_limit": 1,
+ "save_strategy": "epoch",
+ "auto_find_batch_size": false,
+ "mixed_precision": "fp16",
+ "lr": 3e-05,
+ "epochs": 1,
+ "batch_size": 1,
+ "warmup_ratio": 0.1,
+ "gradient_accumulation": 4,
+ "optimizer": "adamw_torch",
+ "scheduler": "linear",
+ "weight_decay": 0.0,
+ "max_grad_norm": 1.0,
+ "seed": 42,
+ "chat_template": "none",
+ "quantization": "int4",
+ "target_modules": "all-linear",
+ "merge_adapter": false,
+ "peft": true,
+ "lora_r": 16,
+ "lora_alpha": 32,
+ "lora_dropout": 0.05,
+ "model_ref": null,
+ "dpo_beta": 0.1,
+ "prompt_text_column": "autotrain_prompt",
+ "text_column": "autotrain_text",
+ "rejected_text_column": "autotrain_rejected_text",
+ "push_to_hub": true,
+ "repo_id": "SungmoKim/gemma-2b-it-adress-bot-seoul-epochs-1",
+ "username": "SungmoKim"
+}