|
import math |
|
import torch |
|
import torch.nn as nn |
|
from einops import repeat |
|
|
|
from timm.models.layers import to_2tuple |
|
|
|
|
|
class PatchEmbed(nn.Module): |
|
""" 2D Image to Patch Embedding |
|
|
|
Image to Patch Embedding using Conv2d |
|
|
|
A convolution based approach to patchifying a 2D image w/ embedding projection. |
|
|
|
Based on the impl in https://github.com/google-research/vision_transformer |
|
|
|
Hacked together by / Copyright 2020 Ross Wightman |
|
|
|
Remove the _assert function in forward function to be compatible with multi-resolution images. |
|
""" |
|
def __init__( |
|
self, |
|
img_size=224, |
|
patch_size=16, |
|
in_chans=3, |
|
embed_dim=768, |
|
norm_layer=None, |
|
flatten=True, |
|
bias=True, |
|
): |
|
super().__init__() |
|
if isinstance(img_size, int): |
|
img_size = to_2tuple(img_size) |
|
elif isinstance(img_size, (tuple, list)) and len(img_size) == 2: |
|
img_size = tuple(img_size) |
|
else: |
|
raise ValueError(f"img_size must be int or tuple/list of length 2. Got {img_size}") |
|
patch_size = to_2tuple(patch_size) |
|
self.img_size = img_size |
|
self.patch_size = patch_size |
|
self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) |
|
self.num_patches = self.grid_size[0] * self.grid_size[1] |
|
self.flatten = flatten |
|
|
|
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias) |
|
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() |
|
|
|
def update_image_size(self, img_size): |
|
self.img_size = img_size |
|
self.grid_size = (img_size[0] // self.patch_size[0], img_size[1] // self.patch_size[1]) |
|
self.num_patches = self.grid_size[0] * self.grid_size[1] |
|
|
|
def forward(self, x): |
|
|
|
|
|
|
|
x = self.proj(x) |
|
if self.flatten: |
|
x = x.flatten(2).transpose(1, 2) |
|
x = self.norm(x) |
|
return x |
|
|
|
|
|
def timestep_embedding(t, dim, max_period=10000, repeat_only=False): |
|
""" |
|
Create sinusoidal timestep embeddings. |
|
:param t: a 1-D Tensor of N indices, one per batch element. |
|
These may be fractional. |
|
:param dim: the dimension of the output. |
|
:param max_period: controls the minimum frequency of the embeddings. |
|
:return: an (N, D) Tensor of positional embeddings. |
|
""" |
|
|
|
if not repeat_only: |
|
half = dim // 2 |
|
freqs = torch.exp( |
|
-math.log(max_period) |
|
* torch.arange(start=0, end=half, dtype=torch.float32) |
|
/ half |
|
).to(device=t.device) |
|
args = t[:, None].float() * freqs[None] |
|
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) |
|
if dim % 2: |
|
embedding = torch.cat( |
|
[embedding, torch.zeros_like(embedding[:, :1])], dim=-1 |
|
) |
|
else: |
|
embedding = repeat(t, "b -> b d", d=dim) |
|
return embedding |
|
|
|
|
|
class TimestepEmbedder(nn.Module): |
|
""" |
|
Embeds scalar timesteps into vector representations. |
|
""" |
|
def __init__(self, hidden_size, frequency_embedding_size=256, out_size=None): |
|
super().__init__() |
|
if out_size is None: |
|
out_size = hidden_size |
|
self.mlp = nn.Sequential( |
|
nn.Linear(frequency_embedding_size, hidden_size, bias=True), |
|
nn.SiLU(), |
|
nn.Linear(hidden_size, out_size, bias=True), |
|
) |
|
self.frequency_embedding_size = frequency_embedding_size |
|
|
|
def forward(self, t): |
|
t_freq = timestep_embedding(t, self.frequency_embedding_size).type(self.mlp[0].weight.dtype) |
|
t_emb = self.mlp(t_freq) |
|
return t_emb |
|
|