LA1512/result

README.md

@@ -1,5 +1,6 @@
 ---
-base_model: LA1512/PubMed-fine-tune
+license: bsd-3-clause
+base_model: pszemraj/led-base-book-summary
 tags:
 - generated_from_trainer
 datasets:
@@ -21,7 +22,7 @@ model-index:
     metrics:
     - name: Rouge1
       type: rouge
-      value: 40.7402
+      value: 42.8608
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -29,14 +30,14 @@ should probably proofread and complete it, then remove this comment. -->
 
 # results
 
-This model is a fine-tuned version of [LA1512/PubMed-fine-tune](https://huggingface.co/LA1512/PubMed-fine-tune) on the pubmed-summarization dataset.
+This model is a fine-tuned version of [pszemraj/led-base-book-summary](https://huggingface.co/pszemraj/led-base-book-summary) on the pubmed-summarization dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.6196
-- Rouge1: 40.7402
-- Rouge2: 16.1978
-- Rougel: 24.4278
-- Rougelsum: 36.5282
-- Gen Len: 179.6185
+- Loss: 3.2597
+- Rouge1: 42.8608
+- Rouge2: 16.655
+- Rougel: 23.8425
+- Rougelsum: 38.0076
+- Gen Len: 273.807
 
 ## Model description
 
@@ -56,22 +57,17 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 1e-05
-- train_batch_size: 8
-- eval_batch_size: 8
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 500
-- num_epochs: 3
+- num_epochs: 1
 - label_smoothing_factor: 0.1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rouge1  | Rouge2  | Rougel  | Rougelsum | Gen Len  |
-|:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:--------:|
-| 3.6132        | 1.0   | 2500 | 3.6766          | 40.5092 | 15.7678 | 24.1228 | 36.3318   | 183.7205 |
-| 3.5939        | 2.0   | 5000 | 3.6276          | 40.7583 | 16.1779 | 24.4375 | 36.5537   | 181.4365 |
-| 3.5419        | 3.0   | 7500 | 3.6196          | 40.7402 | 16.1978 | 24.4278 | 36.5282   | 179.6185 |
 
 
 ### Framework versions

config.json

@@ -1,37 +1,35 @@
 {
-  "_name_or_path": "LA1512/PubMed-fine-tune",
-  "_num_labels": 3,
-  "activation_dropout": 0,
+  "_name_or_path": "pszemraj/led-base-book-summary",
+  "activation_dropout": 0.0,
   "activation_function": "gelu",
-  "add_bias_logits": false,
-  "add_final_layer_norm": false,
   "architectures": [
-    "BartForConditionalGeneration"
+    "LEDForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "attention_window": [
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024
   ],
-  "attention_dropout": 0.1,
   "bos_token_id": 0,
-  "classif_dropout": 0,
-  "classifier_dropout": 0,
-  "d_model": 1024,
-  "decoder_attention_heads": 16,
-  "decoder_ffn_dim": 4096,
-  "decoder_layerdrop": 0,
-  "decoder_layers": 3,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
   "decoder_start_token_id": 2,
   "dropout": 0.1,
   "early_stopping": true,
-  "encoder_attention_heads": 16,
-  "encoder_ffn_dim": 4096,
-  "encoder_layerdrop": 0,
-  "encoder_layers": 12,
+  "encoder_attention_heads": 12,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
   "eos_token_id": 2,
-  "eos_token_ids": [
-    2
-  ],
-  "extra_pos_embeddings": 2,
-  "force_bos_token_to_be_generated": false,
-  "forced_eos_token_id": 2,
-  "gradient_checkpointing": false,
   "id2label": {
     "0": "LABEL_0",
     "1": "LABEL_1",
@@ -44,24 +42,19 @@
     "LABEL_1": 1,
     "LABEL_2": 2
   },
-  "max_length": 62,
-  "max_position_embeddings": 1024,
-  "min_length": 11,
-  "model_type": "bart",
+  "length_penalty": 0.8,
+  "max_decoder_position_embeddings": 1024,
+  "max_encoder_position_embeddings": 16384,
+  "max_length": 1024,
+  "min_length": 8,
+  "model_type": "led",
   "no_repeat_ngram_size": 3,
-  "normalize_before": false,
-  "normalize_embedding": true,
-  "num_beams": 6,
-  "num_hidden_layers": 12,
-  "output_past": true,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
   "pad_token_id": 1,
-  "prefix": " ",
-  "save_step": 58,
-  "scale_embedding": false,
-  "static_position_embeddings": false,
-  "task_specific_params": {},
+  "repetition_penalty": 3.5,
   "torch_dtype": "float32",
   "transformers_version": "4.39.3",
   "use_cache": true,
-  "vocab_size": 50264
+  "vocab_size": 50265
 }

generation_config.json

@@ -3,11 +3,12 @@
   "decoder_start_token_id": 2,
   "early_stopping": true,
   "eos_token_id": 2,
-  "forced_eos_token_id": 2,
-  "max_length": 62,
-  "min_length": 11,
+  "length_penalty": 0.8,
+  "max_length": 1024,
+  "min_length": 8,
   "no_repeat_ngram_size": 3,
-  "num_beams": 6,
+  "num_beams": 4,
   "pad_token_id": 1,
+  "repetition_penalty": 3.5,
   "transformers_version": "4.39.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f34a4a5a93856e5d6532db0bb0e56e053b236b0106766ea844b9d61906ffd50
-size 1020714768
+oid sha256:cdb69fe3cb92cba61864654b5ea695a5e8362996f88f1e41beb60fcdf1a6ea33
+size 647614116

tokenizer.json

@@ -2,13 +2,13 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 256,
+    "max_length": 1024,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
-      "Fixed": 256
+      "Fixed": 1024
     },
     "direction": "Right",
     "pad_to_multiple_of": null,

tokenizer_config.json

@@ -48,10 +48,17 @@
   "eos_token": "</s>",
   "errors": "replace",
   "mask_token": "<mask>",
-  "model_max_length": 1024,
+  "max_length": 1024,
+  "model_max_length": 16384,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "</s>",
-  "tokenizer_class": "BartTokenizer",
+  "stride": 0,
+  "tokenizer_class": "LEDTokenizer",
   "trim_offsets": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8996fa5924b2adc468689d11dc75794fbe5d22d47de3739d11c93031ebbedd55
+oid sha256:301b9e6e7201f138110067138a881ad800c0a1bf46b87ae55423081b9037e7bf
 size 5048