Step 1: Install required libraries

!pip install transformers datasets torch sentencepiece

Step 2: Import Libraries

from datasets import load_dataset from transformers import MarianMTModel, MarianTokenizer import torch from transformers import Trainer, TrainingArguments

Step 3: Load the Dataset

dataset = load_dataset("cfilt/iitb-english-hindi")

Check the structure of the dataset

print(dataset)

Step 4: Prepare Tokenizer and Model

model_name = "Helsinki-NLP/opus-mt-en-hi" tokenizer = MarianTokenizer.from_pretrained(model_name) model = MarianMTModel.from_pretrained(model_name)

Step 5: Preprocess the Dataset

def preprocess_function(examples): # Tokenize the English input and Hindi target model_inputs = tokenizer(examples["en"], truncation=True, padding="max_length", max_length=128) # Tokenize the Hindi target for training with tokenizer.as_target_tokenizer(): labels = tokenizer(examples["hi"], truncation=True, padding="max_length", max_length=128)

model_inputs["labels"] = labels["input_ids"]
return model_inputs

Apply preprocessing to the dataset

tokenized_datasets = dataset.map(preprocess_function, batched=True)

Step 6: Training the Model

training_args = TrainingArguments( output_dir="./results", # output directory for results evaluation_strategy="epoch", # evaluate after every epoch learning_rate=2e-5, # learning rate per_device_train_batch_size=16, # batch size for training per_device_eval_batch_size=16, # batch size for evaluation num_train_epochs=3, # number of training epochs logging_dir="./logs", # directory for storing logs save_steps=500, # save checkpoint every 500 steps )

Initialize the Trainer

trainer = Trainer( model=model, # the pre-trained model args=training_args, # training arguments train_dataset=tokenized_datasets["train"], # training dataset eval_dataset=tokenized_datasets["validation"], # validation dataset )