Fine-tuning + SD penaltyin EURLEX (Level 2)

README.md

@@ -1,3 +1,87 @@
 ---
-license: cc-by-nc-sa-4.0
+widget:
+- text: "KOMMISSIONENS BESLUTNING\naf 6. marts 2006\nom klassificering af visse byggevarers ydeevne med hensyn til reaktion ved brand for så vidt angår trægulve samt vægpaneler og vægbeklædning i massivt træ\n(meddelt under nummer K(2006) 655"
+datasets:
+- multi_eurlex
+metrics:
+- f1
+model-index:
+- name: coastalcph/danish-legal-longformer-eurlex-sd
+  results:
+  - task:
+      type: text-classification
+      name: Danish EURLEX (Level 2)
+    dataset:
+      name: multi_eurlex
+      type: multi_eurlex
+      config: multi_eurlex
+      split: validation
+    metrics:
+    - name: Micro-F1
+      type: micro-f1
+      value: 0.76144
+    - name: Macro-F1
+      type: macro-f1
+      value: 0.52878
 ---
+
+# Model description
+
+This model is a fine-tuned version of [coastalcph/danish-legal-longformer-base](https://huggingface.co/coastalcph/danish-legal-longformer-base) on the Danish part of [MultiEURLEX](https://huggingface.co/datasets/multi_eurlex) dataset using an additional Spectral Decoupling penalty ([Pezeshki et al., 2020](https://arxiv.org/abs/2011.09468).
+
+## Training and evaluation data
+
+The Danish part of [MultiEURLEX](https://huggingface.co/datasets/multi_eurlex) dataset.
+
+## Use of Model
+
+### As a text classifier:
+
+```python
+from transformers import pipeline
+import numpy as np
+
+# Init text classification pipeline
+text_cls_pipe = pipeline(task="text-classification",
+                         model="coastalcph/danish-legal-longformer-eurlex",
+                         use_auth_token='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')
+
+# Encode and Classify document
+predictions = text_cls_pipe("KOMMISSIONENS BESLUTNING\naf 6. marts 2006\nom klassificering af visse byggevarers "
+                            "ydeevne med hensyn til reaktion ved brand for så vidt angår trægulve samt vægpaneler "
+                            "og vægbeklædning i massivt træ\n(meddelt under nummer K(2006) 655")
+
+# Print prediction
+print(predictions)
+# [{'label': 'building and public works', 'score': 0.9626012444496155}]
+```
+
+### As a feature extractor (document embedder):
+
+```python
+from transformers import pipeline
+import numpy as np
+
+# Init feature extraction pipeline
+feature_extraction_pipe = pipeline(task="feature-extraction",
+                                   model="coastalcph/danish-legal-longformer-eurlex",
+                                   use_auth_token='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')
+
+# Encode document
+predictions = feature_extraction_pipe("KOMMISSIONENS BESLUTNING\naf 6. marts 2006\nom klassificering af visse byggevarers "
+                                      "ydeevne med hensyn til reaktion ved brand for så vidt angår trægulve samt vægpaneler "
+                                      "og vægbeklædning i massivt træ\n(meddelt under nummer K(2006) 655")
+
+# Use CLS token representation as document embedding
+document_features = token_wise_features[0][0]
+
+print(document_features.shape)
+# (768,)
+```
+
+## Framework versions
+
+- Transformers 4.18.0
+- Pytorch 1.12.0+cu113
+- Datasets 2.0.0
+- Tokenizers 0.12.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 20.0,
+    "eval_loss": 0.06451725959777832,
+    "eval_macro-f1": 0.5287864667312759,
+    "eval_micro-f1": 0.7614412568306012,
+    "eval_runtime": 85.9935,
+    "eval_samples": 5000,
+    "eval_samples_per_second": 58.144,
+    "eval_steps_per_second": 1.826,
+    "predict_loss": 0.08024133741855621,
+    "predict_macro-f1": 0.46986770055294413,
+    "predict_micro-f1": 0.6993679640782279,
+    "predict_runtime": 86.1285,
+    "predict_samples": 5000,
+    "predict_samples_per_second": 58.053,
+    "predict_steps_per_second": 1.823,
+    "train_loss": 0.03325878010370344,
+    "train_runtime": 59590.7061,
+    "train_samples": 55000,
+    "train_samples_per_second": 18.459,
+    "train_steps_per_second": 0.577
+}

config.json

@@ -0,0 +1,308 @@
+{
+  "_name_or_path": "coastalcph/danish-legal-longformer-base",
+  "architectures": [
+    "LongformerForSequenceClassification"
+  ],
+  "attention_mode": "longformer",
+  "attention_probs_dropout_prob": 0.1,
+  "attention_window": [
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128,
+    128
+  ],
+  "bos_token_id": 1,
+  "classifier_dropout": null,
+  "cls_token_id": 1,
+  "eos_token_id": 2,
+  "finetuning_task": "eurlex-127-concepts",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "health",
+    "1": "social framework",
+    "10": "building and public works",
+    "100": "European Union law",
+    "101": "humanities",
+    "102": "natural and applied sciences",
+    "103": "beverages and sugar",
+    "104": "processed agricultural produce",
+    "105": "agri-foodstuffs",
+    "106": "plant product",
+    "107": "animal product",
+    "108": "foodstuff",
+    "109": "food technology",
+    "11": "chemistry",
+    "110": "regions of EU Member States",
+    "111": "Africa",
+    "112": "overseas countries and territories",
+    "113": "Europe",
+    "114": "America",
+    "115": "economic geography",
+    "116": "Asia and Oceania",
+    "117": "political geography",
+    "118": "economic analysis",
+    "119": "economic conditions",
+    "12": "electronics and electrical engineering",
+    "120": "economic policy",
+    "121": "economic structure",
+    "122": "national accounts",
+    "123": "regions and regional policy",
+    "124": "deterioration of the environment",
+    "125": "natural environment",
+    "126": "environmental policy",
+    "13": "miscellaneous industries",
+    "14": "wood industry",
+    "15": "mechanical engineering",
+    "16": "industrial structures and policy",
+    "17": "leather and textile industries",
+    "18": "insurance",
+    "19": "free movement of capital",
+    "2": "culture and religion",
+    "20": "financing and investment",
+    "21": "prices",
+    "22": "budget",
+    "23": "public finance and budget policy",
+    "24": "monetary economics",
+    "25": "taxation",
+    "26": "financial institutions and credit",
+    "27": "monetary relations",
+    "28": "tariff policy",
+    "29": "trade",
+    "3": "social protection",
+    "30": "trade policy",
+    "31": "marketing",
+    "32": "distributive trades",
+    "33": "consumption",
+    "34": "international trade",
+    "35": "accounting",
+    "36": "competition",
+    "37": "management",
+    "38": "business classification",
+    "39": "legal form of organisations",
+    "4": "migration",
+    "40": "business organisation",
+    "41": "defence",
+    "42": "international affairs",
+    "43": "international security",
+    "44": "cooperation policy",
+    "45": "agricultural activity",
+    "46": "agricultural policy",
+    "47": "forestry",
+    "48": "agricultural structures and production",
+    "49": "cultivation of agricultural land",
+    "5": "family",
+    "50": "means of agricultural production",
+    "51": "fisheries",
+    "52": "farming systems",
+    "53": "production",
+    "54": "research and intellectual property",
+    "55": "technology and technical regulations",
+    "56": "transport policy",
+    "57": "air and space transport",
+    "58": "organisation of transport",
+    "59": "land transport",
+    "6": "construction and town planning",
+    "60": "maritime and inland waterway transport",
+    "61": "organisation of work and working conditions",
+    "62": "labour law and labour relations",
+    "63": "employment",
+    "64": "personnel management and staff remuneration",
+    "65": "labour market",
+    "66": "political party",
+    "67": "parliament",
+    "68": "executive power and public service",
+    "69": "parliamentary proceedings",
+    "7": "demography and population",
+    "70": "electoral procedure and voting",
+    "71": "political framework",
+    "72": "politics and public safety",
+    "73": "sources and branches of the law",
+    "74": "justice",
+    "75": "civil law",
+    "76": "rights and freedoms",
+    "77": "international law",
+    "78": "criminal law",
+    "79": "organisation of the legal system",
+    "8": "social affairs",
+    "80": "information and information processing",
+    "81": "education",
+    "82": "organisation of teaching",
+    "83": "information technology and data processing",
+    "84": "teaching",
+    "85": "documentation",
+    "86": "communications",
+    "87": "world organisations",
+    "88": "United Nations",
+    "89": "extra-European organisations",
+    "9": "iron, steel and other metal industries",
+    "90": "non-governmental organisations",
+    "91": "European organisations",
+    "92": "coal and mining industries",
+    "93": "energy policy",
+    "94": "oil industry",
+    "95": "soft energy",
+    "96": "electrical and nuclear industries",
+    "97": "EU finance",
+    "98": "EU institutions and European civil service",
+    "99": "European construction"
+  },
+  "ignore_attention_mask": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "Africa": 111,
+    "America": 114,
+    "Asia and Oceania": 116,
+    "EU finance": 97,
+    "EU institutions and European civil service": 98,
+    "Europe": 113,
+    "European Union law": 100,
+    "European construction": 99,
+    "European organisations": 91,
+    "United Nations": 88,
+    "accounting": 35,
+    "agri-foodstuffs": 105,
+    "agricultural activity": 45,
+    "agricultural policy": 46,
+    "agricultural structures and production": 48,
+    "air and space transport": 57,
+    "animal product": 107,
+    "beverages and sugar": 103,
+    "budget": 22,
+    "building and public works": 10,
+    "business classification": 38,
+    "business organisation": 40,
+    "chemistry": 11,
+    "civil law": 75,
+    "coal and mining industries": 92,
+    "communications": 86,
+    "competition": 36,
+    "construction and town planning": 6,
+    "consumption": 33,
+    "cooperation policy": 44,
+    "criminal law": 78,
+    "cultivation of agricultural land": 49,
+    "culture and religion": 2,
+    "defence": 41,
+    "demography and population": 7,
+    "deterioration of the environment": 124,
+    "distributive trades": 32,
+    "documentation": 85,
+    "economic analysis": 118,
+    "economic conditions": 119,
+    "economic geography": 115,
+    "economic policy": 120,
+    "economic structure": 121,
+    "education": 81,
+    "electoral procedure and voting": 70,
+    "electrical and nuclear industries": 96,
+    "electronics and electrical engineering": 12,
+    "employment": 63,
+    "energy policy": 93,
+    "environmental policy": 126,
+    "executive power and public service": 68,
+    "extra-European organisations": 89,
+    "family": 5,
+    "farming systems": 52,
+    "financial institutions and credit": 26,
+    "financing and investment": 20,
+    "fisheries": 51,
+    "food technology": 109,
+    "foodstuff": 108,
+    "forestry": 47,
+    "free movement of capital": 19,
+    "health": 0,
+    "humanities": 101,
+    "industrial structures and policy": 16,
+    "information and information processing": 80,
+    "information technology and data processing": 83,
+    "insurance": 18,
+    "international affairs": 42,
+    "international law": 77,
+    "international security": 43,
+    "international trade": 34,
+    "iron, steel and other metal industries": 9,
+    "justice": 74,
+    "labour law and labour relations": 62,
+    "labour market": 65,
+    "land transport": 59,
+    "leather and textile industries": 17,
+    "legal form of organisations": 39,
+    "management": 37,
+    "maritime and inland waterway transport": 60,
+    "marketing": 31,
+    "means of agricultural production": 50,
+    "mechanical engineering": 15,
+    "migration": 4,
+    "miscellaneous industries": 13,
+    "monetary economics": 24,
+    "monetary relations": 27,
+    "national accounts": 122,
+    "natural and applied sciences": 102,
+    "natural environment": 125,
+    "non-governmental organisations": 90,
+    "oil industry": 94,
+    "organisation of teaching": 82,
+    "organisation of the legal system": 79,
+    "organisation of transport": 58,
+    "organisation of work and working conditions": 61,
+    "overseas countries and territories": 112,
+    "parliament": 67,
+    "parliamentary proceedings": 69,
+    "personnel management and staff remuneration": 64,
+    "plant product": 106,
+    "political framework": 71,
+    "political geography": 117,
+    "political party": 66,
+    "politics and public safety": 72,
+    "prices": 21,
+    "processed agricultural produce": 104,
+    "production": 53,
+    "public finance and budget policy": 23,
+    "regions and regional policy": 123,
+    "regions of EU Member States": 110,
+    "research and intellectual property": 54,
+    "rights and freedoms": 76,
+    "social affairs": 8,
+    "social framework": 1,
+    "social protection": 3,
+    "soft energy": 95,
+    "sources and branches of the law": 73,
+    "tariff policy": 28,
+    "taxation": 25,
+    "teaching": 84,
+    "technology and technical regulations": 55,
+    "trade": 29,
+    "trade policy": 30,
+    "transport policy": 56,
+    "wood industry": 14,
+    "world organisations": 87
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2050,
+  "model_max_length": 2048,
+  "model_type": "longformer",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "multi_label_classification",
+  "sep_token_id": 2,
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "use_spectral_decoupling": true,
+  "vocab_size": 32000
+}

predict_results.json

@@ -0,0 +1,9 @@
+{
+    "predict_loss": 0.08024133741855621,
+    "predict_macro-f1": 0.46986770055294413,
+    "predict_micro-f1": 0.6993679640782279,
+    "predict_runtime": 86.1285,
+    "predict_samples": 5000,
+    "predict_samples_per_second": 58.053,
+    "predict_steps_per_second": 1.823
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:053945fba8a2787fc95f1fe319950a81247425aa27a87304be439416a4f9e1af
+size 532736733

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": "<mask>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 2048, "bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": "<mask>", "special_tokens_map_file": "data/PLMs/danish-lm/danish-lex-lm-base/special_tokens_map.json", "name_or_path": "coastalcph/danish-legal-longformer-base", "tokenizer_class": "PreTrainedTokenizerFast"}