Lora Model Uploaded

app.py

@@ -12,7 +12,7 @@ emotions = ["happy", "sad", "angry", "neutral", "fear", "disgust", "surprise"]
 label_mapping = {str(idx): emotion for idx, emotion in enumerate(emotions)}
 
 # Load the trained model
-model_path = "model.pth"
+model_path = "lora_only_model.pth"
 cfg = {
     "model": {
         "encoder": "Wav2Vec2Classifier",
@@ -25,9 +25,17 @@ cfg = {
     }
 }
 model = Wav2Vec2EmotionClassifier(num_classes=len(emotions), optimizer_cfg=cfg["model"]["optimizer"])
-model.load_state_dict(torch.load(model_path, map_location=torch.device("cpu")))
+state_dict = torch.load(model_path, map_location=torch.device("cpu"))
+model.load_state_dict(state_dict, strict=False)
+
+
 model.eval()
 
+
+for name, param in model.named_parameters():
+    if param.requires_grad:
+        print(f"{name}: {param.data}")
+
 # Optional: we define a minimum number of samples to avoid Wav2Vec2 conv errors
 MIN_SAMPLES = 10  # or 16000 if you want at least 1 second
 

encoders/transformer.py

@@ -3,7 +3,7 @@ import torch
 from torchmetrics import Accuracy, Precision, Recall, F1Score
 from transformers import Wav2Vec2Model, Wav2Vec2ForSequenceClassification
 import torch.nn.functional as F
-
+from models.lora import LinearWithLoRA, LoRALayer
 
 class Wav2Vec2Classifier(pl.LightningModule):
     def __init__(self, num_classes, optimizer_cfg = "Adam", l1_lambda=0.0):
@@ -166,6 +166,32 @@ class Wav2Vec2EmotionClassifier(pl.LightningModule):
         else:
             self.optimizer = None
 
+        # Apply LoRA
+        low_rank = 8
+        lora_alpha = 16
+        self.apply_lora(low_rank, lora_alpha)
+
+    def apply_lora(self, rank, alpha):
+        # Replace specific linear layers with LinearWithLoRA
+        for layer in self.model.wav2vec2.encoder.layers:
+            layer.attention.q_proj = LinearWithLoRA(layer.attention.q_proj, rank, alpha)
+            layer.attention.k_proj = LinearWithLoRA(layer.attention.k_proj, rank, alpha)
+            layer.attention.v_proj = LinearWithLoRA(layer.attention.v_proj, rank, alpha)
+            layer.attention.out_proj = LinearWithLoRA(layer.attention.out_proj, rank, alpha)
+
+            layer.feed_forward.intermediate_dense = LinearWithLoRA(layer.feed_forward.intermediate_dense, rank, alpha)
+            layer.feed_forward.output_dense = LinearWithLoRA(layer.feed_forward.output_dense, rank, alpha)
+
+    def state_dict(self, *args, **kwargs):
+        # Save only LoRA and classifier/projector parameters
+        state = super().state_dict(*args, **kwargs)
+        return {k: v for k, v in state.items() if "lora" in k or "classifier" in k or "projector" in k}
+
+    def load_state_dict(self, state_dict, strict=True):
+        missing_keys, unexpected_keys = super().load_state_dict(state_dict, strict=False)
+        if missing_keys or unexpected_keys:
+            print(f"Missing keys: {missing_keys}")
+            print(f"Unexpected keys: {unexpected_keys}")
     def forward(self, x, attention_mask=None):
         return self.model(x, attention_mask=attention_mask).logits
 

lora_only_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc2029a0dcf22d2b626533192bda3fa6098653df84be452b88c4db830a7c9216
+size 8185738

models/__init__.py

@@ -1 +1,2 @@
-from . import CTCencoder
+from . import CTCencoder
+from . import lora

models/lora.py

@@ -0,0 +1,24 @@
+import torch 
+import torch.nn as nn
+
+class LoRALayer(nn.Module):
+    def __init__(self, input_dim, output_dim, rank, alpha):
+        super().__init__()
+        std_dev = 1 / torch.sqrt(torch.tensor(rank).float())
+        self.A = nn.Parameter(torch.randn(input_dim, rank) * std_dev)  # Low-rank matrix A
+        self.B = nn.Parameter(torch.zeros(rank, output_dim))  # Low-rank matrix B
+        self.alpha = alpha  # Scaling factor
+    def forward(self, x):
+        # Apply low-rank adaptation: x + alpha * (x @ A @ B)
+        return self.alpha * (x @ self.A @ self.B)
+    
+
+class LinearWithLoRA(nn.Module):
+    def __init__(self, linear_layer, rank, alpha):
+        super().__init__()
+        self.linear = linear_layer  # Original linear layer
+        self.lora = LoRALayer(linear_layer.in_features, linear_layer.out_features, rank, alpha)  # LoRA layer
+
+    def forward(self, x):
+        # Combine original linear layer output with LoRA adaptation
+        return self.linear(x) + self.lora(x)