finnstrom3693
/

mini-sun-init-bert-tf-110m

Model card Files Files and versions Community

finnstrom3693 commited on Oct 1

Commit

ad1f612

•

1 Parent(s): 121d5a6

Update modeling-dev.py

Files changed (1) hide show

modeling-dev.py +1 -1

modeling-dev.py CHANGED Viewed

@@ -49,7 +49,7 @@ class MiniSunModel(tf.keras.Model):
         super(MiniSunModel, self).build(input_shape)
     def _build_decoder_block(self):
-        # Decoder block with multi-query attention and feed-forward layers, using RMSNorm and regularization
         return [
             layers.MultiHeadAttention(num_heads=self.config.num_attention_heads, key_dim=self.config.hidden_size,
                                       kernel_initializer=initializers.he_normal(),

         super(MiniSunModel, self).build(input_shape)
     def _build_decoder_block(self):
+        # Decoder block with multi-query attention and feed-forward layers, using regularization
         return [
             layers.MultiHeadAttention(num_heads=self.config.num_attention_heads, key_dim=self.config.hidden_size,
                                       kernel_initializer=initializers.he_normal(),