|
from turtle import forward |
|
import torch |
|
from torch import nn |
|
import torch.nn.functional as F |
|
import numpy as np |
|
|
|
class ConvNextLayer(nn.Module): |
|
def __init__(self, dim, depth=4): |
|
super().__init__() |
|
self.net = nn.Sequential( |
|
*[ConvNextBlock(dim=dim) for j in range(depth)] |
|
) |
|
|
|
def forward(self, x): |
|
return self.net(x) |
|
|
|
def compute_params(self): |
|
num = 0 |
|
for param in self.parameters(): |
|
num += np.prod(param.size()) |
|
|
|
return num |
|
|
|
class ConvNextBlock(nn.Module): |
|
r""" ConvNeXt Block. There are two equivalent implementations: |
|
(1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) |
|
(2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back |
|
We use (2) as we find it slightly faster in PyTorch |
|
|
|
Args: |
|
dim (int): Number of input channels. |
|
drop_path (float): Stochastic depth rate. Default: 0.0 |
|
layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. |
|
""" |
|
def __init__(self, dim, layer_scale_init_value=1e-6): |
|
super().__init__() |
|
self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) |
|
self.norm = LayerNorm(dim, eps=1e-6) |
|
self.pwconv1 = nn.Linear(dim, 4 * dim) |
|
self.act = nn.GELU() |
|
self.pwconv2 = nn.Linear(4 * dim, dim) |
|
self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), |
|
requires_grad=True) if layer_scale_init_value > 0 else None |
|
|
|
|
|
|
|
def forward(self, x): |
|
input = x |
|
x = self.dwconv(x) |
|
x = x.permute(0, 2, 3, 1) |
|
x = self.norm(x) |
|
x = self.pwconv1(x) |
|
x = self.act(x) |
|
x = self.pwconv2(x) |
|
if self.gamma is not None: |
|
x = self.gamma * x |
|
x = x.permute(0, 3, 1, 2) |
|
|
|
x = input + x |
|
return x |
|
|
|
|
|
class LayerNorm(nn.Module): |
|
r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. |
|
The ordering of the dimensions in the inputs. channels_last corresponds to inputs with |
|
shape (batch_size, height, width, channels) while channels_first corresponds to inputs |
|
with shape (batch_size, channels, height, width). |
|
""" |
|
def __init__(self, normalized_shape, eps=1e-6, data_format="channels_last"): |
|
super().__init__() |
|
self.weight = nn.Parameter(torch.ones(normalized_shape)) |
|
self.bias = nn.Parameter(torch.zeros(normalized_shape)) |
|
self.eps = eps |
|
self.data_format = data_format |
|
if self.data_format not in ["channels_last", "channels_first"]: |
|
raise NotImplementedError |
|
self.normalized_shape = (normalized_shape, ) |
|
|
|
def forward(self, x): |
|
if self.data_format == "channels_last": |
|
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) |
|
elif self.data_format == "channels_first": |
|
u = x.mean(1, keepdim=True) |
|
s = (x - u).pow(2).mean(1, keepdim=True) |
|
x = (x - u) / torch.sqrt(s + self.eps) |
|
x = self.weight[:, None, None] * x + self.bias[:, None, None] |
|
return x |