Update README.md

README.md

@@ -1,3 +1,55 @@
----
-license: mit
----
+# Step 1: Install required libraries
+!pip install transformers datasets torch sentencepiece
+
+# Step 2: Import Libraries
+from datasets import load_dataset
+from transformers import MarianMTModel, MarianTokenizer
+import torch
+from transformers import Trainer, TrainingArguments
+
+# Step 3: Load the Dataset
+dataset = load_dataset("cfilt/iitb-english-hindi")
+
+# Check the structure of the dataset
+print(dataset)
+
+# Step 4: Prepare Tokenizer and Model
+model_name = "Helsinki-NLP/opus-mt-en-hi"
+tokenizer = MarianTokenizer.from_pretrained(model_name)
+model = MarianMTModel.from_pretrained(model_name)
+
+# Step 5: Preprocess the Dataset
+def preprocess_function(examples):
+    # Tokenize the English input and Hindi target
+    model_inputs = tokenizer(examples["en"], truncation=True, padding="max_length", max_length=128)
+    # Tokenize the Hindi target for training
+    with tokenizer.as_target_tokenizer():
+        labels = tokenizer(examples["hi"], truncation=True, padding="max_length", max_length=128)
+    
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+# Apply preprocessing to the dataset
+tokenized_datasets = dataset.map(preprocess_function, batched=True)
+
+# Step 6: Training the Model
+training_args = TrainingArguments(
+    output_dir="./results",          # output directory for results
+    evaluation_strategy="epoch",     # evaluate after every epoch
+    learning_rate=2e-5,              # learning rate
+    per_device_train_batch_size=16,  # batch size for training
+    per_device_eval_batch_size=16,   # batch size for evaluation
+    num_train_epochs=3,              # number of training epochs
+    logging_dir="./logs",            # directory for storing logs
+    save_steps=500,                  # save checkpoint every 500 steps
+)
+
+# Initialize the Trainer
+trainer = Trainer(
+    model=model,                         # the pre-trained model
+    args=training_args,                  # training arguments
+    train_dataset=tokenized_datasets["train"],   # training dataset
+    eval_dataset=tokenized_datasets["validation"],  # validation dataset
+)
+
+# Train