Upload BiLSTM

blstm_config.py

@@ -0,0 +1,17 @@
+from transformers import PretrainedConfig
+
+import torch
+
+class BiLSTMConfig(PretrainedConfig):
+    def __init__(self, vocab_size=23626, embed_dim=100,
+                 num_layers=1, hidden_dim=256, dropout=0.33,
+                 output_dim=128, predict_output=10, **kwargs):
+        
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.embed_dim = embed_dim
+        self.num_layers = num_layers
+        self.hidden_dim = hidden_dim
+        self.dropout = dropout
+        self.output_dim = output_dim
+        self.predict_output = predict_output

blstm_model.py

@@ -0,0 +1,42 @@
+from transformers import PreTrainedModel
+
+from torch import nn
+import torch
+
+class BiLSTM(PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.hidden_dim = config.hidden_dim
+        self.predict_output = config.predict_output
+
+        self.embed_layer = nn.Embedding(config.vocab_size, config.embed_dim, padding_idx=0)
+        self.biLSTM = nn.LSTM(input_size=config.embed_dim,
+                              hidden_size=config.hidden_dim // 2, # BiLSTM will concatenate the 2 directional LSTMs
+                              num_layers=config.num_layers,
+                              bidirectional=True,
+                              batch_first=True)
+        self.linear = nn.Linear(config.hidden_dim, config.output_dim)
+        self.dropout = nn.Dropout(config.dropout)
+        self.elu = nn.ELU()
+        self.fc = nn.Linear(config.output_dim, config.predict_output)
+        # self.device_ = config.device
+    
+    def forward(self, input):   # input is a list of indices, shape batch_size, seq_len
+        x = self.embed_layer(input)                     # batch_size, seq_len, 100  (This is only when batch_first=True!!!!)
+        batch_size = x.size(0)
+        hidden, cell = self.init_hidden(batch_size)
+
+        out, hidden = self.biLSTM(x, (hidden, cell))    # seq_len, batch_size, (hidden_dim//2) * 2
+        
+        out = self.dropout(out)
+
+        out = self.elu(self.linear(out))                # self.linear(out): batch_size, seq_len, output_dim
+        
+        out = self.fc(out)
+        
+        return out, hidden
+    
+    def init_hidden(self, batch_size):
+        hidden = torch.zeros(2, batch_size, self.hidden_dim//2)
+        cell = torch.zeros(2, batch_size, self.hidden_dim//2)
+        return hidden, cell

config.json

@@ -0,0 +1,18 @@
+{
+  "architectures": [
+    "BiLSTM"
+  ],
+  "auto_map": {
+    "AutoConfig": "blstm_config.BiLSTMConfig",
+    "AutoModel": "blstm_model.BiLSTM"
+  },
+  "dropout": 0.33,
+  "embed_dim": 100,
+  "hidden_dim": 256,
+  "num_layers": 1,
+  "output_dim": 128,
+  "predict_output": 10,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.2",
+  "vocab_size": 23626
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d69933282398681daa6dfa3e9596c1e96707ceac15a031ce20cc2cc3b10fd9b
+size 10533208