Create distillbert-baseline.py

distillbert-baseline.py

@@ -0,0 +1,68 @@
+from datasets import load_dataset
+from transformers import TrainingArguments
+from transformers import DistilBertForSequenceClassification, DistilBertTokenizerFast
+
+
+
+
+dataset = load_dataset("quotaclimat/frugalaichallenge-text-train")
+
+# %% [markdown]
+# 
+
+# %%
+LABEL_MAPPING = {
+        "0_not_relevant": 0,
+        "1_not_happening": 1,
+        "2_not_human": 2,
+        "3_not_bad": 3,
+        "4_solutions_harmful_unnecessary": 4,
+        "5_science_unreliable": 5,
+        "6_proponents_biased": 6,
+        "7_fossil_fuels_needed": 7
+    }
+
+# %%
+dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
+
+# %%
+print(dataset)
+
+# %%
+
+tokenizer = DistilBertTokenizerFast.from_pretrained("distilbert-base-uncased")
+# Tokenize the datasets
+def tokenize_function(examples):
+    return tokenizer(examples["quote"], padding="max_length", truncation=True)
+
+train_dataset = dataset["train"].map(tokenize_function, batched=True)
+test_dataset = dataset["test"].map(tokenize_function, batched=True)
+
+
+model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=8)  # Set num_labels for your classification task
+
+# %%
+
+# Define training arguments
+training_args = TrainingArguments(
+    output_dir="./results",            # Output directory for saved models
+    eval_strategy="epoch",       # Evaluation strategy (can be "steps" or "epoch")
+    per_device_train_batch_size=16,    # Batch size for training
+    per_device_eval_batch_size=64,     # Batch size for evaluation
+    num_train_epochs=3,                # Number of training epochs
+    logging_dir="./logs",              # Directory for logs
+    logging_steps=10,                  # How often to log
+)
+
+
+# %%
+
+trainer = Trainer(
+    model=model,                       # The model to train
+    args=training_args,                # The training arguments
+    train_dataset=train_dataset,       # The training dataset
+    eval_dataset=test_dataset          # The evaluation dataset
+)
+trainer.train()
+results = trainer.evaluate()
+print(results)