Update app.py

app.py

@@ -1,13 +1,37 @@
 import torch
 import pandas as pd
-from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification, TrainingArguments 
+from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification, TrainingArguments, Trainer
 import gradio as gr
 from gradio.mix import Parallel, Series
 #import torch.nn.functional as F
 from aitextgen import aitextgen
+
 from datasets import load_dataset
 dataset = load_dataset("bananabot/engMollywoodSummaries")
-dataset
+tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-1.3B")
+
+def tokenize_function(examples):
+    return tokenizer(examples["text"], padding="max_length", truncation=True)
+tokenized_datasets = dataset.map(tokenize_function, batched=True)
+
+model = AutoModelForCausalLM.from_pretrained("EleutherAI/gpt-neo-1.3B").to(device)
+training_args = TrainingArguments(output_dir="test_trainer")
+small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
+small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    return metric.compute(predictions=predictions, references=labels)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=small_train_dataset,
+    eval_dataset=small_eval_dataset,
+    compute_metrics=compute_metrics,
+)
+
+trainer.train()
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
@@ -15,7 +39,7 @@ ai = aitextgen(model="EleutherAI/gpt-neo-1.3B")
 
 #model_name = "EleutherAI/gpt-neo-125M"
 #tokenizer = AutoTokenizer.from_pretrained(model_name)
-#model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
+
 
 #max_length=123
 #input_txt = "This malayalam movie is about"