hflqf88888 commited on
Commit
1e7b654
1 Parent(s): 0694c08
config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "OdysseyAgent",
3
+ "architectures": [
4
+ "QWenLMHeadModel"
5
+ ],
6
+ "attn_dropout_prob": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_qwen.QWenConfig",
9
+ "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
10
+ },
11
+ "bf16": true,
12
+ "emb_dropout_prob": 0.0,
13
+ "fp16": false,
14
+ "fp32": false,
15
+ "hidden_size": 4096,
16
+ "his_len": 4,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 22016,
19
+ "kv_channels": 128,
20
+ "layer_norm_epsilon": 1e-06,
21
+ "max_position_embeddings": 8192,
22
+ "model_type": "qwen",
23
+ "no_bias": true,
24
+ "num_attention_heads": 32,
25
+ "num_hidden_layers": 32,
26
+ "onnx_safe": null,
27
+ "rotary_emb_base": 10000,
28
+ "rotary_pct": 1.0,
29
+ "scale_attn_weights": true,
30
+ "seq_length": 2048,
31
+ "tie_word_embeddings": false,
32
+ "tokenizer_type": "QWenTokenizer",
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.32.0",
35
+ "use_cache": true,
36
+ "use_dynamic_ntk": true,
37
+ "use_flash_attn": false,
38
+ "use_logn_attn": true,
39
+ "visual": {
40
+ "heads": 16,
41
+ "image_size": 448,
42
+ "image_start_id": 151857,
43
+ "layers": 48,
44
+ "mlp_ratio": 4.9231,
45
+ "output_dim": 4096,
46
+ "patch_size": 14,
47
+ "width": 1664
48
+ },
49
+ "vocab_size": 151936
50
+ }
configuration_qwen.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from transformers import PretrainedConfig
7
+
8
+
9
+ class QWenConfig(PretrainedConfig):
10
+ model_type = "qwen"
11
+ keys_to_ignore_at_inference = ["past_key_values"]
12
+
13
+ def __init__(
14
+ self,
15
+ vocab_size=151936,
16
+ hidden_size=4096,
17
+ num_hidden_layers=32,
18
+ num_attention_heads=32,
19
+ emb_dropout_prob=0.0,
20
+ attn_dropout_prob=0.0,
21
+ layer_norm_epsilon=1e-6,
22
+ initializer_range=0.02,
23
+ max_position_embeddings=8192,
24
+ scale_attn_weights=True,
25
+ use_cache=True,
26
+ bf16=False,
27
+ fp16=False,
28
+ fp32=False,
29
+ kv_channels=128,
30
+ rotary_pct=1.0,
31
+ rotary_emb_base=10000,
32
+ use_dynamic_ntk=True,
33
+ use_logn_attn=True,
34
+ use_flash_attn="auto",
35
+ intermediate_size=22016,
36
+ no_bias=True,
37
+ tie_word_embeddings=False,
38
+ **kwargs,
39
+ ):
40
+ self.vocab_size = vocab_size
41
+ self.hidden_size = hidden_size
42
+ self.intermediate_size = intermediate_size
43
+ self.num_hidden_layers = num_hidden_layers
44
+ self.num_attention_heads = num_attention_heads
45
+ self.emb_dropout_prob = emb_dropout_prob
46
+ self.attn_dropout_prob = attn_dropout_prob
47
+ self.layer_norm_epsilon = layer_norm_epsilon
48
+ self.initializer_range = initializer_range
49
+ self.scale_attn_weights = scale_attn_weights
50
+ self.use_cache = use_cache
51
+ self.max_position_embeddings = max_position_embeddings
52
+ self.bf16 = bf16
53
+ self.fp16 = fp16
54
+ self.fp32 = fp32
55
+ self.kv_channels = kv_channels
56
+ self.rotary_pct = rotary_pct
57
+ self.rotary_emb_base = rotary_emb_base
58
+ self.use_dynamic_ntk = use_dynamic_ntk
59
+ self.use_logn_attn = use_logn_attn
60
+ self.use_flash_attn = use_flash_attn
61
+ self.no_bias = no_bias
62
+ super().__init__(
63
+ tie_word_embeddings=tie_word_embeddings,
64
+ **kwargs
65
+ )
generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "transformers_version": "4.32.0"
4
+ }
modeling_qwen.py ADDED
@@ -0,0 +1,1421 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ print('OdysseyAgent')
7
+
8
+ import importlib
9
+ import math
10
+ from typing import TYPE_CHECKING, Optional, Tuple, Union, Callable, List, Any, Generator
11
+ import os, json
12
+ import numpy as np
13
+ import torch
14
+ import torch.nn.functional as F
15
+ import torch.utils.checkpoint
16
+ from torch.cuda.amp import autocast
17
+
18
+ from torch.nn import CrossEntropyLoss
19
+ from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
20
+ from transformers.generation.logits_process import LogitsProcessorList
21
+
22
+ if TYPE_CHECKING:
23
+ from transformers.generation.streamers import BaseStreamer
24
+ from transformers.generation.utils import GenerateOutput
25
+ from transformers.modeling_outputs import (
26
+ BaseModelOutputWithPast,
27
+ CausalLMOutputWithPast,
28
+ )
29
+ from transformers.modeling_utils import PreTrainedModel
30
+ from transformers.utils import logging
31
+
32
+ try:
33
+ from einops import rearrange
34
+ except ImportError:
35
+ rearrange = None
36
+ from torch import nn
37
+
38
+ SUPPORT_CUDA = torch.cuda.is_available()
39
+ SUPPORT_BF16 = SUPPORT_CUDA and torch.cuda.is_bf16_supported()
40
+ SUPPORT_FP16 = SUPPORT_CUDA and torch.cuda.get_device_capability(0)[0] >= 7
41
+
42
+ from torch.nn.init import trunc_normal_
43
+ import sys
44
+ sys.path.append('../OdysseyAgent')
45
+
46
+ from configuration_qwen import QWenConfig
47
+ from qwen_generation_utils import (
48
+ HistoryType,
49
+ make_context,
50
+ decode_tokens,
51
+ get_stop_words_ids,
52
+ StopWordsLogitsProcessor,
53
+ )
54
+ from visual import VisionTransformer
55
+
56
+ IMAGE_HISTORY = '../data/his_index.json'
57
+
58
+ USE_RESAMPLER = True
59
+
60
+ print(IMAGE_HISTORY)
61
+ logger = logging.get_logger(__name__)
62
+
63
+ _CHECKPOINT_FOR_DOC = "qwen"
64
+ _CONFIG_FOR_DOC = "QWenConfig"
65
+
66
+ QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ["qwen-7b"]
67
+
68
+ _ERROR_BAD_CHAT_FORMAT = """\
69
+ We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
70
+ If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
71
+ 我们检测到您可能在使用预训练模型(而非chat模型)进行多轮chat,因为您当前在generation_config指定的chat_format,并未设置为我们在对话中所支持的"chatml"格式。
72
+ 如果您在直接使用我们从Huggingface提供的模型,请确保您在调用model.chat()时,使用的是"Qwen/Qwen-7B-Chat"模型(而非"Qwen/Qwen-7B"预训练模型)。
73
+ """
74
+
75
+ _SENTINEL = object()
76
+ _ERROR_STREAM_IN_CHAT = """\
77
+ Pass argument `stream` to model.chat() is buggy, deprecated, and marked for removal. Please use model.chat_stream(...) instead of model.chat(..., stream=True).
78
+ 向model.chat()传入参数stream的用法可能存在Bug,该用法已被废弃,将在未来被移除。请使用model.chat_stream(...)代替model.chat(..., stream=True)。
79
+ """
80
+
81
+ apply_rotary_emb_func = None
82
+ rms_norm = None
83
+
84
+
85
+ # Copied from transformers.models.bart.modeling_bart._make_causal_mask
86
+ def _make_causal_mask(
87
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
88
+ ):
89
+ """
90
+ Make causal mask used for bi-directional self-attention.
91
+ """
92
+ bsz, tgt_len = input_ids_shape
93
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
94
+ mask_cond = torch.arange(mask.size(-1), device=device)
95
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
96
+ mask = mask.to(dtype)
97
+
98
+ if past_key_values_length > 0:
99
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
100
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
101
+
102
+
103
+ # Copied from transformers.models.bart.modeling_bart._expand_mask
104
+ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
105
+ """
106
+ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
107
+ """
108
+ bsz, src_len = mask.size()
109
+ tgt_len = tgt_len if tgt_len is not None else src_len
110
+
111
+ expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
112
+
113
+ inverted_mask = 1.0 - expanded_mask
114
+
115
+ return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
116
+
117
+ def get_abs_pos(abs_pos, tgt_size):
118
+ # abs_pos: L, C
119
+ # tgt_size: M
120
+ # return: M, C
121
+ src_size = int(math.sqrt(abs_pos.size(0)))
122
+ tgt_size = int(math.sqrt(tgt_size))
123
+ dtype = abs_pos.dtype
124
+
125
+ if src_size != tgt_size:
126
+ return F.interpolate(
127
+ abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
128
+ size=(tgt_size, tgt_size),
129
+ mode="bicubic",
130
+ align_corners=False,
131
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
132
+ else:
133
+ return abs_pos
134
+
135
+
136
+ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
137
+ """
138
+ grid_size: int of the grid height and width
139
+ return:
140
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
141
+ """
142
+ grid_h = np.arange(grid_size, dtype=np.float32)
143
+ grid_w = np.arange(grid_size, dtype=np.float32)
144
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
145
+ grid = np.stack(grid, axis=0)
146
+
147
+ grid = grid.reshape([2, 1, grid_size, grid_size])
148
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
149
+ if cls_token:
150
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
151
+ return pos_embed
152
+
153
+
154
+ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
155
+ assert embed_dim % 2 == 0
156
+
157
+ # use half of dimensions to encode grid_h
158
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
159
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
160
+
161
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
162
+ return emb
163
+
164
+
165
+ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
166
+ """
167
+ embed_dim: output dimension for each position
168
+ pos: a list of positions to be encoded: size (M,)
169
+ out: (M, D)
170
+ """
171
+ assert embed_dim % 2 == 0
172
+ omega = np.arange(embed_dim // 2, dtype=np.float32)
173
+ omega /= embed_dim / 2.
174
+ omega = 1. / 10000**omega # (D/2,)
175
+
176
+ pos = pos.reshape(-1) # (M,)
177
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
178
+
179
+ emb_sin = np.sin(out) # (M, D/2)
180
+ emb_cos = np.cos(out) # (M, D/2)
181
+
182
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
183
+ return emb
184
+
185
+
186
+
187
+ class HisResampler(nn.Module):
188
+ """
189
+ A 2D perceiver-resampler network with one cross attention layers by
190
+ (grid_size**2) learnable queries and 2d sincos pos_emb
191
+ Outputs:
192
+ A tensor with the shape of (grid_size**2, embed_dim)
193
+ """
194
+ def __init__(
195
+ self,
196
+ embed_dim=4096,
197
+ num_heads=32,
198
+ grid_size=16,
199
+ kv_dim=None,
200
+ norm_layer=nn.LayerNorm
201
+ ):
202
+ super().__init__()
203
+ self.num_queries = grid_size ** 2
204
+ self.embed_dim = embed_dim
205
+ self.num_heads = num_heads
206
+
207
+ self.pos_embed = nn.Parameter(
208
+ torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
209
+ ).requires_grad_(False)
210
+
211
+ self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
212
+ trunc_normal_(self.query, std=.02)
213
+
214
+ if kv_dim is not None and kv_dim != embed_dim:
215
+ self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
216
+ else:
217
+ self.kv_proj = nn.Identity()
218
+
219
+ self.attn = nn.MultiheadAttention(embed_dim, num_heads)
220
+ self.ln_q = norm_layer(embed_dim)
221
+ self.ln_kv = norm_layer(embed_dim)
222
+
223
+ self.ln_post = norm_layer(embed_dim)
224
+ self.proj = nn.Parameter((embed_dim** -0.5) * torch.randn(embed_dim, embed_dim))
225
+
226
+ self.apply(self._init_weights)
227
+
228
+ def _init_weights(self, m):
229
+ if isinstance(m, nn.Linear):
230
+ trunc_normal_(m.weight, std=.02)
231
+ if isinstance(m, nn.Linear) and m.bias is not None:
232
+ nn.init.constant_(m.bias, 0)
233
+ elif isinstance(m, nn.LayerNorm):
234
+ nn.init.constant_(m.bias, 0)
235
+ nn.init.constant_(m.weight, 1.0)
236
+
237
+ def forward(self, x, attn_mask=None):
238
+
239
+ #pos_embed = get_abs_pos(self.pos_embed, x.size(1))
240
+ x = self.kv_proj(x)
241
+ x = self.ln_kv(x).permute(1, 0, 2)
242
+
243
+ N = x.shape[1]
244
+ q = self.ln_q(self.query)
245
+ out = self.attn(
246
+ self._repeat(q, N),# + self.pos_embed.unsqueeze(1),
247
+ x, # + pos_embed.unsqueeze(1),
248
+ x,
249
+ attn_mask=attn_mask)[0]
250
+ out = out.permute(1, 0, 2)
251
+ out = self.ln_post(out)
252
+ out = out @ self.proj
253
+ return out
254
+
255
+ def _repeat(self, query, N: int):
256
+ return query.unsqueeze(1).repeat(1, N, 1)
257
+
258
+
259
+
260
+ class QWenAttention(nn.Module):
261
+ def __init__(self, config):
262
+ super().__init__()
263
+
264
+ self.register_buffer("masked_bias", torch.tensor(-1e4), persistent=False)
265
+ self.seq_length = config.seq_length
266
+
267
+ self.hidden_size = config.hidden_size
268
+ self.split_size = config.hidden_size
269
+ self.num_heads = config.num_attention_heads
270
+ self.head_dim = self.hidden_size // self.num_heads
271
+
272
+ self.scale_attn_weights = True
273
+
274
+ self.projection_size = config.kv_channels * config.num_attention_heads
275
+
276
+ assert self.projection_size % config.num_attention_heads == 0
277
+ self.hidden_size_per_attention_head = (
278
+ self.projection_size // config.num_attention_heads
279
+ )
280
+
281
+ self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size)
282
+
283
+ self.c_proj = nn.Linear(
284
+ config.hidden_size, self.projection_size, bias=not config.no_bias
285
+ )
286
+
287
+ self.is_fp32 = not (config.bf16 or config.fp16)
288
+ self.bf16 = config.bf16
289
+
290
+ self.use_dynamic_ntk = config.use_dynamic_ntk
291
+ self.use_logn_attn = config.use_logn_attn
292
+
293
+ logn_list = [
294
+ math.log(i, self.seq_length) if i > self.seq_length else 1
295
+ for i in range(1, 32768)
296
+ ]
297
+ self.logn_tensor = torch.tensor(logn_list)[None, :, None, None]
298
+
299
+ self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
300
+
301
+ def _attn(self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None):
302
+ attn_weights = torch.matmul(query, key.transpose(-1, -2))
303
+
304
+ if self.scale_attn_weights:
305
+ attn_weights = attn_weights / torch.full(
306
+ [],
307
+ value.size(-1) ** 0.5,
308
+ dtype=attn_weights.dtype,
309
+ device=attn_weights.device,
310
+ )
311
+
312
+ query_length, key_length = query.size(-2), key.size(-2)
313
+ # causal_mask = self.bias[
314
+ # :, :, key_length - query_length : key_length, :key_length
315
+ # ]
316
+ # mask_value = torch.finfo(attn_weights.dtype).min
317
+ # mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to(
318
+ # attn_weights.device
319
+ # )
320
+ # attn_weights = torch.where(
321
+ # causal_mask, attn_weights.to(attn_weights.dtype), mask_value
322
+ # )
323
+ attn_weights = attn_weights + attention_mask
324
+
325
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
326
+
327
+ attn_weights = attn_weights.type(value.dtype)
328
+ attn_weights = self.attn_dropout(attn_weights)
329
+
330
+ if head_mask is not None:
331
+ attn_weights = attn_weights * head_mask
332
+
333
+ attn_output = torch.matmul(attn_weights, value)
334
+ attn_output = attn_output.transpose(1, 2)
335
+
336
+ return attn_output, attn_weights
337
+
338
+ def _upcast_and_reordered_attn(
339
+ self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None
340
+ ):
341
+ bsz, num_heads, q_seq_len, dk = query.size()
342
+ _, _, k_seq_len, _ = key.size()
343
+
344
+ attn_weights = torch.empty(
345
+ bsz * num_heads,
346
+ q_seq_len,
347
+ k_seq_len,
348
+ dtype=torch.float32,
349
+ device=query.device,
350
+ )
351
+
352
+ scale_factor = 1.0
353
+ if self.scale_attn_weights:
354
+ scale_factor /= float(value.size(-1)) ** 0.5
355
+
356
+ with autocast(enabled=False):
357
+ q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(
358
+ -1, dk, k_seq_len
359
+ )
360
+ attn_weights = torch.baddbmm(
361
+ attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor
362
+ )
363
+ attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)
364
+
365
+ query_length, key_length = query.size(-2), key.size(-2)
366
+ causal_mask = registered_causal_mask[
367
+ :, :, key_length - query_length : key_length, :key_length
368
+ ]
369
+ mask_value = torch.finfo(attn_weights.dtype).min
370
+ mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(
371
+ attn_weights.device
372
+ )
373
+ attn_weights = torch.where(causal_mask, attn_weights, mask_value)
374
+
375
+ if attention_mask is not None:
376
+ attn_weights = attn_weights + attention_mask
377
+
378
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
379
+
380
+ if attn_weights.dtype != torch.float32:
381
+ raise RuntimeError(
382
+ "Error with upcasting, attn_weights does not have dtype torch.float32"
383
+ )
384
+ attn_weights = attn_weights.type(value.dtype)
385
+ attn_weights = self.attn_dropout(attn_weights)
386
+
387
+ if head_mask is not None:
388
+ attn_weights = attn_weights * head_mask
389
+
390
+ attn_output = torch.matmul(attn_weights, value)
391
+
392
+ return attn_output, attn_weights
393
+
394
+ def _split_heads(self, tensor, num_heads, attn_head_size):
395
+ new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
396
+ tensor = tensor.view(new_shape)
397
+ return tensor
398
+
399
+ def _merge_heads(self, tensor, num_heads, attn_head_size):
400
+ tensor = tensor.contiguous()
401
+ new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
402
+ return tensor.view(new_shape)
403
+
404
+ def forward(
405
+ self,
406
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
407
+ rotary_pos_emb: Optional[List[torch.Tensor]] = None,
408
+ registered_causal_mask: Optional[torch.Tensor] = None,
409
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
410
+ attention_mask: Optional[torch.FloatTensor] = None,
411
+ head_mask: Optional[torch.FloatTensor] = None,
412
+ encoder_hidden_states: Optional[torch.Tensor] = None,
413
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
414
+ output_attentions: Optional[bool] = False,
415
+ use_cache: Optional[bool] = False,
416
+ ):
417
+
418
+ mixed_x_layer = self.c_attn(hidden_states)
419
+
420
+ query, key, value = mixed_x_layer.split(self.split_size, dim=2)
421
+
422
+ query = self._split_heads(query, self.num_heads, self.head_dim)
423
+ key = self._split_heads(key, self.num_heads, self.head_dim)
424
+ value = self._split_heads(value, self.num_heads, self.head_dim)
425
+
426
+ if rotary_pos_emb is not None:
427
+ cur_len = query.shape[1]
428
+ rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
429
+ rotary_pos_emb = (rotary_pos_emb,) * 2
430
+ q_pos_emb, k_pos_emb = rotary_pos_emb
431
+ # Slice the pos emb for current inference
432
+ query = apply_rotary_pos_emb(query, q_pos_emb)
433
+ key = apply_rotary_pos_emb(key, k_pos_emb)
434
+
435
+ if layer_past is not None:
436
+ past_key, past_value = layer_past[0], layer_past[1]
437
+ key = torch.cat((past_key, key), dim=1)
438
+ value = torch.cat((past_value, value), dim=1)
439
+
440
+ if use_cache:
441
+ present = (key, value)
442
+ else:
443
+ present = None
444
+
445
+ if self.use_logn_attn and not self.training:
446
+ if self.logn_tensor.device != query.device or self.logn_tensor.dtype != query.dtype:
447
+ self.logn_tensor = self.logn_tensor.to(query.device).type_as(query)
448
+ seq_start = key.size(1) - query.size(1)
449
+ seq_end = key.size(1)
450
+ logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :]
451
+ query = query * logn_tensor.expand_as(query)
452
+
453
+ query = query.permute(0, 2, 1, 3)
454
+ key = key.permute(0, 2, 1, 3)
455
+ value = value.permute(0, 2, 1, 3)
456
+ attn_output, attn_weight = self._attn(
457
+ query, key, value, registered_causal_mask, attention_mask, head_mask
458
+ )
459
+ context_layer = self._merge_heads(
460
+ attn_output, self.num_heads, self.head_dim
461
+ )
462
+
463
+ attn_output = self.c_proj(context_layer)
464
+
465
+ outputs = (attn_output, present)
466
+ if output_attentions:
467
+ outputs += (attn_weight,)
468
+
469
+ return outputs
470
+
471
+
472
+ class QWenMLP(nn.Module):
473
+ def __init__(self, config):
474
+ super().__init__()
475
+ self.w1 = nn.Linear(
476
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
477
+ )
478
+ self.w2 = nn.Linear(
479
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
480
+ )
481
+ ff_dim_in = config.intermediate_size // 2
482
+ self.c_proj = nn.Linear(ff_dim_in, config.hidden_size, bias=not config.no_bias)
483
+
484
+ def forward(self, hidden_states):
485
+ a1 = self.w1(hidden_states)
486
+ a2 = self.w2(hidden_states)
487
+ intermediate_parallel = a1 * F.silu(a2)
488
+ output = self.c_proj(intermediate_parallel)
489
+ return output
490
+
491
+ class QWenBlock(nn.Module):
492
+ def __init__(self, config):
493
+ super().__init__()
494
+ hidden_size = config.hidden_size
495
+ self.bf16 = config.bf16
496
+
497
+ self.ln_1 = RMSNorm(
498
+ hidden_size,
499
+ eps=config.layer_norm_epsilon,
500
+ )
501
+ self.attn = QWenAttention(config)
502
+ self.ln_2 = RMSNorm(
503
+ hidden_size,
504
+ eps=config.layer_norm_epsilon,
505
+ )
506
+
507
+ self.mlp = QWenMLP(config)
508
+
509
+ def forward(
510
+ self,
511
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
512
+ rotary_pos_emb: Optional[List[torch.Tensor]] = None,
513
+ registered_causal_mask: Optional[torch.Tensor] = None,
514
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
515
+ attention_mask: Optional[torch.FloatTensor] = None,
516
+ head_mask: Optional[torch.FloatTensor] = None,
517
+ encoder_hidden_states: Optional[torch.Tensor] = None,
518
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
519
+ use_cache: Optional[bool] = False,
520
+ output_attentions: Optional[bool] = False,
521
+ ):
522
+ layernorm_output = self.ln_1(hidden_states)
523
+
524
+ attn_outputs = self.attn(
525
+ layernorm_output,
526
+ rotary_pos_emb,
527
+ registered_causal_mask=registered_causal_mask,
528
+ layer_past=layer_past,
529
+ attention_mask=attention_mask,
530
+ head_mask=head_mask,
531
+ use_cache=use_cache,
532
+ output_attentions=output_attentions,
533
+ )
534
+ attn_output = attn_outputs[0]
535
+
536
+ outputs = attn_outputs[1:]
537
+
538
+ residual = hidden_states
539
+ layernorm_input = attn_output + residual
540
+
541
+ layernorm_output = self.ln_2(layernorm_input)
542
+
543
+ residual = layernorm_input
544
+ mlp_output = self.mlp(layernorm_output)
545
+ hidden_states = residual + mlp_output
546
+
547
+ if use_cache:
548
+ outputs = (hidden_states,) + outputs
549
+ else:
550
+ outputs = (hidden_states,) + outputs[1:]
551
+
552
+ return outputs
553
+
554
+
555
+ class QWenPreTrainedModel(PreTrainedModel):
556
+ config_class = QWenConfig
557
+ base_model_prefix = "transformer"
558
+ is_parallelizable = False
559
+ supports_gradient_checkpointing = True
560
+ _no_split_modules = ["QWenBlock"]
561
+
562
+ def __init__(self, *inputs, **kwargs):
563
+ super().__init__(*inputs, **kwargs)
564
+
565
+ def _init_weights(self, module):
566
+ """Initialize the weights."""
567
+ if isinstance(module, nn.Linear):
568
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
569
+ if module.bias is not None:
570
+ module.bias.data.zero_()
571
+ elif isinstance(module, nn.Embedding):
572
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
573
+ if module.padding_idx is not None:
574
+ module.weight.data[module.padding_idx].zero_()
575
+ elif isinstance(module, RMSNorm):
576
+ module.weight.data.fill_(1.0)
577
+
578
+ for name, p in module.named_parameters():
579
+ if name == "c_proj.weight":
580
+ p.data.normal_(
581
+ mean=0.0,
582
+ std=(
583
+ self.config.initializer_range
584
+ / math.sqrt(2 * self.config.num_hidden_layers)
585
+ ),
586
+ )
587
+
588
+ def _set_gradient_checkpointing(self, module, value=False):
589
+ if isinstance(module, QWenModel):
590
+ module.gradient_checkpointing = value
591
+
592
+
593
+ class QWenModel(QWenPreTrainedModel):
594
+ _keys_to_ignore_on_load_missing = ["attn.masked_bias"]
595
+
596
+ def __init__(self, config):
597
+ super().__init__(config)
598
+ self.his_len = config.his_len
599
+ self.vocab_size = config.vocab_size
600
+ self.num_hidden_layers = config.num_hidden_layers
601
+ self.embed_dim = config.hidden_size
602
+
603
+ self.gradient_checkpointing = False
604
+ self.use_dynamic_ntk = config.use_dynamic_ntk
605
+ self.seq_length = config.seq_length
606
+
607
+ self.wte = nn.Embedding(self.vocab_size, self.embed_dim)
608
+
609
+ self.drop = nn.Dropout(config.emb_dropout_prob)
610
+
611
+ if config.rotary_pct == 1.0:
612
+ self.rotary_ndims = None
613
+ else:
614
+ assert config.rotary_pct < 1
615
+ self.rotary_ndims = int(
616
+ config.kv_channels * config.rotary_pct
617
+ )
618
+ dim = (
619
+ self.rotary_ndims
620
+ if self.rotary_ndims is not None
621
+ else config.kv_channels
622
+ )
623
+ self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
624
+
625
+ self.use_flash_attn = config.use_flash_attn
626
+ self.is_fp32 = not (config.bf16 or config.fp16)
627
+ self.registered_causal_mask = None
628
+
629
+ self.h = nn.ModuleList(
630
+ [
631
+ QWenBlock(
632
+ config
633
+ )
634
+ for i in range(config.num_hidden_layers)
635
+ ]
636
+ )
637
+ self.ln_f = RMSNorm(
638
+ self.embed_dim,
639
+ eps=config.layer_norm_epsilon,
640
+ )
641
+
642
+ self.visual = VisionTransformer(**config.visual)
643
+
644
+ self.post_init()
645
+
646
+ if USE_RESAMPLER:
647
+ print('init RESAMPLER')
648
+ self.his_resampler = HisResampler()
649
+
650
+ self.imgtoken_dict = {}
651
+ if os.path.isdir(IMAGE_HISTORY):
652
+ for subdata in os.listdir(IMAGE_HISTORY):
653
+ sub_img_dict = json.load(open(os.path.join(IMAGE_HISTORY, subdata)))
654
+ self.imgtoken_dict.update(sub_img_dict)
655
+ else:
656
+ self.imgtoken_dict = json.load(open(IMAGE_HISTORY))
657
+
658
+ print('imgtoken_dict cache len:', len(self.imgtoken_dict))
659
+
660
+ def set_input_embeddings(self, new_embeddings):
661
+ self.wte = new_embeddings
662
+
663
+ # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
664
+ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
665
+ # create causal mask
666
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
667
+ combined_attention_mask = None
668
+ if input_shape[-1] > 1:
669
+ combined_attention_mask = _make_causal_mask(
670
+ input_shape,
671
+ inputs_embeds.dtype,
672
+ device=inputs_embeds.device,
673
+ past_key_values_length=past_key_values_length,
674
+ )
675
+
676
+ if attention_mask is not None:
677
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
678
+ expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
679
+ inputs_embeds.device
680
+ )
681
+ combined_attention_mask = (
682
+ expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
683
+ )
684
+
685
+ return combined_attention_mask
686
+
687
+
688
+ def forward(
689
+ self,
690
+ input_ids: Optional[torch.LongTensor] = None,
691
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
692
+ attention_mask: Optional[torch.FloatTensor] = None,
693
+ token_type_ids: Optional[torch.LongTensor] = None,
694
+ position_ids: Optional[torch.LongTensor] = None,
695
+ head_mask: Optional[torch.FloatTensor] = None,
696
+ inputs_embeds: Optional[torch.FloatTensor] = None,
697
+ encoder_hidden_states: Optional[torch.Tensor] = None,
698
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
699
+ use_cache: Optional[bool] = None,
700
+ output_attentions: Optional[bool] = None,
701
+ output_hidden_states: Optional[bool] = None,
702
+ return_dict: Optional[bool] = None,
703
+ ):
704
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
705
+
706
+ if past_key_values is None and torch.any(input_ids == self.config.visual['image_start_id']):
707
+ bos_pos = torch.where(input_ids == self.config.visual['image_start_id'])
708
+ eos_pos = torch.where(input_ids == self.config.visual['image_start_id'] + 1)
709
+ assert (bos_pos[0] == eos_pos[0]).all()
710
+ img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1)
711
+ now_images = []
712
+ his_images = []
713
+ C_list = []
714
+ images = []
715
+ his_idx = []
716
+ his_image_temp = []
717
+ for idx, (i, a, b) in enumerate(img_pos):
718
+ image = input_ids[i][a + 1 : b - 1].tolist()
719
+ image = image[ : image.index(self.config.visual['image_start_id'] + 2)]
720
+ image_path = bytes(image).decode('utf-8')
721
+
722
+ if image_path.startswith('image-history: '):
723
+ his_idx.append(idx)
724
+ image_path = image_path.replace('image-history: ', '')
725
+ his_list = self.imgtoken_dict[image_path][-self.his_len:] # t0 - tn-1
726
+ assert len(his_list) > 0, his_list
727
+
728
+ his_images.extend(his_list)
729
+ his_image_temp.append(his_list)
730
+
731
+ else:
732
+ now_images.append(image_path)
733
+
734
+ now_images = self.visual.encode(now_images)
735
+
736
+ if len(his_images) > 0:
737
+ his_images = self.visual.encode(his_images)
738
+ his_tkn = None
739
+
740
+ start_pos = 0
741
+ for his_scr in his_image_temp:
742
+ his_len = len(his_scr)
743
+ his_img_feature = his_images[start_pos: start_pos + his_len] # [b, l, d]
744
+ if USE_RESAMPLER:
745
+ his_img_feature = his_img_feature.reshape(1, -1, his_img_feature.size(-1))
746
+ his_vis_tkn = self.his_resampler(his_img_feature) # [l, d]
747
+ else:
748
+ raise ValueError("You cannot run without History Redsampler!")
749
+ his_tkn = his_vis_tkn if his_tkn is None else torch.concat((his_tkn, his_vis_tkn), dim=0)
750
+ start_pos += his_len
751
+ assert start_pos == len(his_images)
752
+ his_images = his_tkn
753
+
754
+ now_p, his_p = 0, 0
755
+ for j in range(len(img_pos)):
756
+ if j not in his_idx:
757
+ images.append(now_images[now_p])
758
+ now_p += 1
759
+ else:
760
+ images.append(his_images[his_p])
761
+ his_p += 1
762
+ images = torch.stack(images, dim=0)
763
+ assert len(images) == len(img_pos) == len(now_images) + len(his_images)
764
+
765
+ fake_images = None
766
+ elif self.training:
767
+ fake_images=torch.zeros(1,3,224,224).to(
768
+ dtype=self.visual.conv1.weight.dtype, device=self.visual.conv1.weight.device)
769
+ images = self.visual(fake_images)
770
+ else:
771
+ fake_images = None
772
+ images = None
773
+
774
+ output_attentions = (
775
+ output_attentions
776
+ if output_attentions is not None
777
+ else self.config.output_attentions
778
+ )
779
+ output_hidden_states = (
780
+ output_hidden_states
781
+ if output_hidden_states is not None
782
+ else self.config.output_hidden_states
783
+ )
784
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
785
+ return_dict = (
786
+ return_dict if return_dict is not None else self.config.use_return_dict
787
+ )
788
+
789
+ if input_ids is not None and inputs_embeds is not None:
790
+ raise ValueError(
791
+ "You cannot specify both input_ids and inputs_embeds at the same time"
792
+ )
793
+ elif input_ids is not None:
794
+ input_shape = input_ids.size()
795
+ input_ids = input_ids.view(-1, input_shape[-1])
796
+ batch_size = input_ids.shape[0]
797
+ elif inputs_embeds is not None:
798
+ input_shape = inputs_embeds.size()[:-1]
799
+ batch_size = inputs_embeds.shape[0]
800
+ else:
801
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
802
+
803
+
804
+ if token_type_ids is not None:
805
+ token_type_ids = token_type_ids.view(-1, input_shape[-1])
806
+ if position_ids is not None:
807
+ position_ids = position_ids.view(-1, input_shape[-1])
808
+
809
+ if past_key_values is None:
810
+ past_length = 0
811
+ past_key_values = tuple([None] * len(self.h))
812
+ else:
813
+ past_length = past_key_values[0][0].size(-2)
814
+
815
+ if position_ids is None:
816
+ position_ids = torch.arange(
817
+ past_length,
818
+ input_shape[-1] + past_length,
819
+ dtype=torch.long,
820
+ device=device,
821
+ )
822
+ position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
823
+
824
+ encoder_attention_mask = None
825
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
826
+
827
+ if inputs_embeds is None:
828
+ inputs_embeds = self.wte(input_ids)
829
+
830
+ if batch_size <= 0:
831
+ raise ValueError("batch_size has to be defined and > 0")
832
+ attention_mask = self._prepare_decoder_attention_mask(
833
+ attention_mask, input_shape, inputs_embeds, past_length
834
+ )
835
+
836
+ hidden_states = inputs_embeds
837
+
838
+ kv_seq_len = hidden_states.size()[1]
839
+ if past_key_values[0] is not None:
840
+ # past key values[0][0] shape: bs * seq_len * head_num * dim
841
+ kv_seq_len += past_key_values[0][0].shape[1]
842
+ if (
843
+ self.use_dynamic_ntk
844
+ and kv_seq_len == hidden_states.size()[1]
845
+ and not self.training
846
+ ):
847
+ context_value = math.log(kv_seq_len / self.seq_length, 2) + 1
848
+ ntk_alpha = 2 ** math.ceil(context_value) - 1
849
+ ntk_alpha = max(ntk_alpha, 1)
850
+ else:
851
+ ntk_alpha = self.rotary_emb._ntk_alpha_cached
852
+
853
+ rotary_pos_emb = self.rotary_emb(kv_seq_len, ntk_alpha=ntk_alpha)
854
+ for idx in range(len(rotary_pos_emb)):
855
+ rotary_pos_emb[idx] = rotary_pos_emb[idx].to(hidden_states.device)
856
+ hidden_states = self.drop(hidden_states).clone()
857
+
858
+ if fake_images is not None:
859
+ hidden_states = hidden_states + images.mean()*0
860
+ elif images is not None:
861
+ for idx, (i, a, b) in enumerate(img_pos):
862
+ hidden_states[i][a + 1 : b] = images[idx]
863
+ output_shape = input_shape + (hidden_states.size(-1),)
864
+
865
+ if self.gradient_checkpointing and self.training:
866
+ if use_cache:
867
+ logger.warning_once(
868
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
869
+ )
870
+ use_cache = False
871
+
872
+ presents = () if use_cache else None
873
+ all_self_attentions = () if output_attentions else None
874
+ all_hidden_states = () if output_hidden_states else None
875
+ for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
876
+
877
+ if output_hidden_states:
878
+ all_hidden_states = all_hidden_states + (hidden_states,)
879
+
880
+ if self.gradient_checkpointing and self.training:
881
+
882
+ def create_custom_forward(module):
883
+ def custom_forward(*inputs):
884
+ # None for past_key_value
885
+ return module(*inputs, use_cache, output_attentions)
886
+
887
+ return custom_forward
888
+
889
+ outputs = torch.utils.checkpoint.checkpoint(
890
+ create_custom_forward(block),
891
+ hidden_states,
892
+ rotary_pos_emb,
893
+ self.registered_causal_mask,
894
+ None,
895
+ attention_mask,
896
+ head_mask[i],
897
+ encoder_hidden_states,
898
+ encoder_attention_mask,
899
+ )
900
+ else:
901
+ outputs = block(
902
+ hidden_states,
903
+ layer_past=layer_past,
904
+ rotary_pos_emb=rotary_pos_emb,
905
+ registered_causal_mask=self.registered_causal_mask,
906
+ attention_mask=attention_mask,
907
+ head_mask=head_mask[i],
908
+ encoder_hidden_states=encoder_hidden_states,
909
+ encoder_attention_mask=encoder_attention_mask,
910
+ use_cache=use_cache,
911
+ output_attentions=output_attentions,
912
+ )
913
+
914
+ hidden_states = outputs[0]
915
+ if use_cache is True:
916
+ presents = presents + (outputs[1],)
917
+
918
+ if output_attentions:
919
+ all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
920
+
921
+ hidden_states = self.ln_f(hidden_states)
922
+ hidden_states = hidden_states.view(output_shape)
923
+ # Add last hidden state
924
+ if output_hidden_states:
925
+ all_hidden_states = all_hidden_states + (hidden_states,)
926
+
927
+ if not return_dict:
928
+ return tuple(
929
+ v for v in [hidden_states, presents, all_hidden_states] if v is not None
930
+ )
931
+
932
+ return BaseModelOutputWithPast(
933
+ last_hidden_state=hidden_states,
934
+ past_key_values=presents,
935
+ hidden_states=all_hidden_states,
936
+ attentions=all_self_attentions,
937
+ )
938
+
939
+
940
+ class QWenLMHeadModel(QWenPreTrainedModel):
941
+ _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.rotary_emb\.inv_freq"]
942
+ _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.masked_bias"]
943
+
944
+ def __init__(self, config):
945
+ super().__init__(config)
946
+ assert (
947
+ config.bf16 + config.fp16 + config.fp32 <= 1
948
+ ), "Only one of \"bf16\", \"fp16\", \"fp32\" can be true"
949
+
950
+ autoset_precision = config.bf16 + config.fp16 + config.fp32 == 0
951
+
952
+ if autoset_precision:
953
+ if SUPPORT_BF16:
954
+ logger.warn(
955
+ "The model is automatically converting to bf16 for faster inference. "
956
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
957
+ )
958
+ config.bf16 = True
959
+ elif SUPPORT_FP16:
960
+ logger.warn(
961
+ "The model is automatically converting to fp16 for faster inference. "
962
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
963
+ )
964
+ config.fp16 = True
965
+ else:
966
+ config.fp32 = True
967
+
968
+ if config.bf16 and SUPPORT_CUDA and not SUPPORT_BF16:
969
+ logger.warn("Your device does NOT seem to support bf16, you can switch to fp16 or fp32 by by passing fp16/fp32=True in \"AutoModelForCausalLM.from_pretrained\".")
970
+ if config.fp16 and SUPPORT_CUDA and not SUPPORT_FP16:
971
+ logger.warn("Your device does NOT support faster inference with fp16, please switch to fp32 which is likely to be faster")
972
+ if config.fp32:
973
+ if SUPPORT_BF16:
974
+ logger.warn("Your device support faster inference by passing bf16=True in \"AutoModelForCausalLM.from_pretrained\".")
975
+ elif SUPPORT_FP16:
976
+ logger.warn("Your device support faster inference by passing fp16=True in \"AutoModelForCausalLM.from_pretrained\".")
977
+
978
+ self.transformer = QWenModel(config)
979
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
980
+
981
+ if config.bf16:
982
+ self.transformer.bfloat16()
983
+ self.lm_head.bfloat16()
984
+ if config.fp16:
985
+ self.transformer.half()
986
+ self.lm_head.half()
987
+ self.post_init()
988
+
989
+ def get_output_embeddings(self):
990
+ return self.lm_head
991
+
992
+ def set_output_embeddings(self, new_embeddings):
993
+ self.lm_head = new_embeddings
994
+
995
+ def prepare_inputs_for_generation(
996
+ self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs
997
+ ):
998
+ token_type_ids = kwargs.get("token_type_ids", None)
999
+ if past_key_values:
1000
+ input_ids = input_ids[:, -1].unsqueeze(-1)
1001
+ if token_type_ids is not None:
1002
+ token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
1003
+
1004
+ attention_mask = kwargs.get("attention_mask", None)
1005
+ position_ids = kwargs.get("position_ids", None)
1006
+
1007
+ if attention_mask is not None and position_ids is None:
1008
+ position_ids = attention_mask.long().cumsum(-1) - 1
1009
+ position_ids.masked_fill_(attention_mask == 0, 1)
1010
+ if past_key_values:
1011
+ position_ids = position_ids[:, -1].unsqueeze(-1)
1012
+ else:
1013
+ position_ids = None
1014
+
1015
+ if inputs_embeds is not None and past_key_values is None:
1016
+ model_inputs = {"inputs_embeds": inputs_embeds}
1017
+ else:
1018
+ model_inputs = {"input_ids": input_ids}
1019
+
1020
+ model_inputs.update(
1021
+ {
1022
+ "past_key_values": past_key_values,
1023
+ "use_cache": kwargs.get("use_cache"),
1024
+ "position_ids": position_ids,
1025
+ "attention_mask": attention_mask,
1026
+ "token_type_ids": token_type_ids,
1027
+ }
1028
+ )
1029
+ return model_inputs
1030
+
1031
+ def forward(
1032
+ self,
1033
+ input_ids: Optional[torch.LongTensor] = None,
1034
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
1035
+ attention_mask: Optional[torch.FloatTensor] = None,
1036
+ token_type_ids: Optional[torch.LongTensor] = None,
1037
+ position_ids: Optional[torch.LongTensor] = None,
1038
+ head_mask: Optional[torch.FloatTensor] = None,
1039
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1040
+ encoder_hidden_states: Optional[torch.Tensor] = None,
1041
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1042
+ labels: Optional[torch.LongTensor] = None,
1043
+ use_cache: Optional[bool] = None,
1044
+ output_attentions: Optional[bool] = None,
1045
+ output_hidden_states: Optional[bool] = None,
1046
+ return_dict: Optional[bool] = None,
1047
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1048
+
1049
+ return_dict = (
1050
+ return_dict if return_dict is not None else self.config.use_return_dict
1051
+ )
1052
+
1053
+ transformer_outputs = self.transformer(
1054
+ input_ids,
1055
+ past_key_values=past_key_values,
1056
+ attention_mask=attention_mask,
1057
+ token_type_ids=token_type_ids,
1058
+ position_ids=position_ids,
1059
+ head_mask=head_mask,
1060
+ inputs_embeds=inputs_embeds,
1061
+ encoder_hidden_states=encoder_hidden_states,
1062
+ encoder_attention_mask=encoder_attention_mask,
1063
+ use_cache=use_cache,
1064
+ output_attentions=output_attentions,
1065
+ output_hidden_states=output_hidden_states,
1066
+ return_dict=return_dict,
1067
+ )
1068
+ hidden_states = transformer_outputs[0]
1069
+
1070
+ lm_logits = self.lm_head(hidden_states)
1071
+
1072
+ loss = None
1073
+ if labels is not None:
1074
+ labels = labels.to(lm_logits.device)
1075
+ shift_logits = lm_logits[..., :-1, :].contiguous()
1076
+ shift_labels = labels[..., 1:].contiguous()
1077
+ loss_fct = CrossEntropyLoss()
1078
+ loss = loss_fct(
1079
+ shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
1080
+ )
1081
+
1082
+ if not return_dict:
1083
+ output = (lm_logits,) + transformer_outputs[1:]
1084
+ return ((loss,) + output) if loss is not None else output
1085
+
1086
+ return CausalLMOutputWithPast(
1087
+ loss=loss,
1088
+ logits=lm_logits,
1089
+ past_key_values=transformer_outputs.past_key_values,
1090
+ hidden_states=transformer_outputs.hidden_states,
1091
+ attentions=transformer_outputs.attentions,
1092
+ )
1093
+
1094
+ @staticmethod
1095
+ def _reorder_cache(
1096
+ past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
1097
+ ) -> Tuple[Tuple[torch.Tensor]]:
1098
+
1099
+ return tuple(
1100
+ tuple(
1101
+ past_state.index_select(0, beam_idx.to(past_state.device))
1102
+ for past_state in layer_past
1103
+ )
1104
+ for layer_past in past_key_values
1105
+ )
1106
+
1107
+ def chat(
1108
+ self,
1109
+ tokenizer: PreTrainedTokenizer,
1110
+ query: str,
1111
+ history: Optional[HistoryType],
1112
+ system: str = "You are a helpful assistant.",
1113
+ append_history: bool = True,
1114
+ stream: Optional[bool] = _SENTINEL,
1115
+ stop_words_ids: Optional[List[List[int]]] = None,
1116
+ generation_config: Optional[GenerationConfig] = None,
1117
+ **kwargs,
1118
+ ) -> Tuple[str, HistoryType]:
1119
+ generation_config = generation_config if generation_config is not None else self.generation_config
1120
+
1121
+ assert stream is _SENTINEL, _ERROR_STREAM_IN_CHAT
1122
+ assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1123
+ if history is None:
1124
+ history = []
1125
+ if stop_words_ids is None:
1126
+ stop_words_ids = []
1127
+
1128
+ max_window_size = kwargs.get('max_window_size', None)
1129
+ if max_window_size is None:
1130
+ max_window_size = generation_config.max_window_size
1131
+ raw_text, context_tokens = make_context(
1132
+ tokenizer,
1133
+ query,
1134
+ history=history,
1135
+ system=system,
1136
+ max_window_size=max_window_size,
1137
+ chat_format=generation_config.chat_format,
1138
+ )
1139
+
1140
+ stop_words_ids.extend(get_stop_words_ids(
1141
+ generation_config.chat_format, tokenizer
1142
+ ))
1143
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1144
+ outputs = self.generate(
1145
+ input_ids,
1146
+ stop_words_ids=stop_words_ids,
1147
+ return_dict_in_generate=False,
1148
+ generation_config=generation_config,
1149
+ **kwargs,
1150
+ )
1151
+
1152
+ response = decode_tokens(
1153
+ outputs[0],
1154
+ tokenizer,
1155
+ raw_text_len=len(raw_text),
1156
+ context_length=len(context_tokens),
1157
+ chat_format=generation_config.chat_format,
1158
+ verbose=False,
1159
+ errors='replace'
1160
+ )
1161
+
1162
+ if append_history:
1163
+ history.append((query, response))
1164
+
1165
+ return response, history
1166
+
1167
+ def chat_stream(
1168
+ self,
1169
+ tokenizer: PreTrainedTokenizer,
1170
+ query: str,
1171
+ history: Optional[HistoryType],
1172
+ system: str = "You are a helpful assistant.",
1173
+ stop_words_ids: Optional[List[List[int]]] = None,
1174
+ logits_processor: Optional[LogitsProcessorList] = None,
1175
+ generation_config: Optional[GenerationConfig] = None,
1176
+ **kwargs,
1177
+ ) -> Generator[str, Any, None]:
1178
+ generation_config = generation_config if generation_config is not None else self.generation_config
1179
+ assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1180
+ if history is None:
1181
+ history = []
1182
+ if stop_words_ids is None:
1183
+ stop_words_ids = []
1184
+
1185
+ max_window_size = kwargs.get('max_window_size', None)
1186
+ if max_window_size is None:
1187
+ max_window_size = generation_config.max_window_size
1188
+ raw_text, context_tokens = make_context(
1189
+ tokenizer,
1190
+ query,
1191
+ history=history,
1192
+ system=system,
1193
+ max_window_size=max_window_size,
1194
+ chat_format=generation_config.chat_format,
1195
+ )
1196
+
1197
+ stop_words_ids.extend(get_stop_words_ids(
1198
+ generation_config.chat_format, tokenizer
1199
+ ))
1200
+ if stop_words_ids is not None:
1201
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1202
+ stop_words_ids=stop_words_ids,
1203
+ eos_token_id=generation_config.eos_token_id,
1204
+ )
1205
+ if logits_processor is None:
1206
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1207
+ else:
1208
+ logits_processor.append(stop_words_logits_processor)
1209
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1210
+
1211
+ from transformers_stream_generator.main import NewGenerationMixin, StreamGenerationConfig
1212
+ self.__class__.generate_stream = NewGenerationMixin.generate
1213
+ self.__class__.sample_stream = NewGenerationMixin.sample_stream
1214
+ stream_config = StreamGenerationConfig(**generation_config.to_dict(), do_stream=True)
1215
+
1216
+ def stream_generator():
1217
+ outputs = []
1218
+ for token in self.generate_stream(
1219
+ input_ids,
1220
+ return_dict_in_generate=False,
1221
+ generation_config=stream_config,
1222
+ logits_processor=logits_processor,
1223
+ seed=-1,
1224
+ **kwargs):
1225
+ outputs.append(token.item())
1226
+ yield tokenizer.decode(outputs, skip_special_tokens=True, errors='ignore', keep_image_special=True)
1227
+
1228
+ return stream_generator()
1229
+
1230
+ def generate(
1231
+ self,
1232
+ inputs: Optional[torch.Tensor] = None,
1233
+ generation_config: Optional[GenerationConfig] = None,
1234
+ logits_processor: Optional[LogitsProcessorList] = None,
1235
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1236
+ prefix_allowed_tokens_fn: Optional[
1237
+ Callable[[int, torch.Tensor], List[int]]
1238
+ ] = None,
1239
+ synced_gpus: Optional[bool] = None,
1240
+ assistant_model: Optional["PreTrainedModel"] = None,
1241
+ streamer: Optional["BaseStreamer"] = None,
1242
+ **kwargs,
1243
+ ) -> Union[GenerateOutput, torch.LongTensor]:
1244
+ generation_config = generation_config if generation_config is not None else self.generation_config
1245
+
1246
+ # Process stop_words_ids.
1247
+ stop_words_ids = kwargs.pop("stop_words_ids", None)
1248
+ if stop_words_ids is None and generation_config is not None:
1249
+ stop_words_ids = getattr(generation_config, "stop_words_ids", None)
1250
+ if stop_words_ids is None:
1251
+ stop_words_ids = getattr(generation_config, "stop_words_ids", None)
1252
+
1253
+ if stop_words_ids is not None:
1254
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1255
+ stop_words_ids=stop_words_ids,
1256
+ eos_token_id=generation_config.eos_token_id,
1257
+ )
1258
+ if logits_processor is None:
1259
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1260
+ else:
1261
+ logits_processor.append(stop_words_logits_processor)
1262
+
1263
+ return super().generate(
1264
+ inputs,
1265
+ generation_config=generation_config,
1266
+ logits_processor=logits_processor,
1267
+ stopping_criteria=stopping_criteria,
1268
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1269
+ synced_gpus=synced_gpus,
1270
+ assistant_model=assistant_model,
1271
+ streamer=streamer,
1272
+ **kwargs,
1273
+ )
1274
+
1275
+
1276
+ class RotaryEmbedding(torch.nn.Module):
1277
+ def __init__(self, dim, base=10000):
1278
+ super().__init__()
1279
+ self.dim = dim
1280
+ self.base = base
1281
+ self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
1282
+ if importlib.util.find_spec("einops") is None:
1283
+ raise RuntimeError("einops is required for Rotary Embedding")
1284
+
1285
+ self._rotary_pos_emb_cache = None
1286
+ self._seq_len_cached = 0
1287
+ self._ntk_alpha_cached = 1.0
1288
+
1289
+ def update_rotary_pos_emb_cache(self, max_seq_len, offset=0, ntk_alpha=1.0):
1290
+ seqlen = max_seq_len + offset
1291
+ if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached:
1292
+ base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
1293
+ self.inv_freq = 1.0 / (
1294
+ base
1295
+ ** (
1296
+ torch.arange(0, self.dim, 2, device=self.inv_freq.device).float()
1297
+ / self.dim
1298
+ )
1299
+ )
1300
+ self._seq_len_cached = max(2 * seqlen, 16)
1301
+ self._ntk_alpha_cached = ntk_alpha
1302
+ seq = torch.arange(self._seq_len_cached, device=self.inv_freq.device)
1303
+ freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq)
1304
+
1305
+ emb = torch.cat((freqs, freqs), dim=-1)
1306
+ from einops import rearrange
1307
+
1308
+ emb = rearrange(emb, "n d -> 1 n 1 d")
1309
+
1310
+ cos, sin = emb.cos(), emb.sin()
1311
+ self._rotary_pos_emb_cache = [cos, sin]
1312
+
1313
+ def forward(self, max_seq_len, offset=0, ntk_alpha=1.0):
1314
+ self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha)
1315
+ cos, sin = self._rotary_pos_emb_cache
1316
+ return [cos[:, offset : offset + max_seq_len], sin[:, offset : offset + max_seq_len]]
1317
+
1318
+
1319
+ def _rotate_half(x):
1320
+ from einops import rearrange
1321
+
1322
+ x = rearrange(x, "... (j d) -> ... j d", j=2)
1323
+ x1, x2 = x.unbind(dim=-2)
1324
+ return torch.cat((-x2, x1), dim=-1)
1325
+
1326
+
1327
+ def apply_rotary_pos_emb(t, freqs):
1328
+ cos, sin = freqs
1329
+ if apply_rotary_emb_func is not None and t.is_cuda:
1330
+ t_ = t.float()
1331
+ cos = cos.squeeze(0).squeeze(1)[:, : cos.shape[-1] // 2]
1332
+ sin = sin.squeeze(0).squeeze(1)[:, : sin.shape[-1] // 2]
1333
+ output = apply_rotary_emb_func(t_, cos, sin).type_as(t)
1334
+ return output
1335
+ else:
1336
+ rot_dim = freqs[0].shape[-1]
1337
+ cos, sin = freqs
1338
+ t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:]
1339
+ t_ = t_.float()
1340
+ t_pass_ = t_pass_.float()
1341
+ t_ = (t_ * cos) + (_rotate_half(t_) * sin)
1342
+ return torch.cat((t_, t_pass_), dim=-1).type_as(t)
1343
+
1344
+
1345
+ class RMSNorm(torch.nn.Module):
1346
+ def __init__(self, dim: int, eps: float = 1e-6):
1347
+ super().__init__()
1348
+ self.eps = eps
1349
+ self.weight = nn.Parameter(torch.ones(dim))
1350
+
1351
+ def _norm(self, x):
1352
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
1353
+
1354
+ def forward(self, x):
1355
+ if rms_norm is not None and x.is_cuda:
1356
+ return rms_norm(x, self.weight, self.eps)
1357
+ else:
1358
+ output = self._norm(x.float()).type_as(x)
1359
+ return output * self.weight
1360
+
1361
+
1362
+ if __name__ == '__main__':
1363
+ import json
1364
+ from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
1365
+ model_path = '/mnt/petrelfs/luquanfeng/Gui-Copilot/Qwenvl-chat'
1366
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
1367
+ tokenizer.padding_side = 'left'
1368
+ tokenizer.pad_token_id = tokenizer.eod_id
1369
+ qs = ["Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/2971401471600307458_9.png</img>\nPlease show the next action. Goal: What's the price of the new iPhone on eBay?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/2971401471600307458_9.png</img>\nPrevious Actions: TYPE: n ebay\nCLICK: (393, 113)\nSCROLL: UP\nSCROLL: UP\nCLICK: (356, 599)\nCLICK: (454, 914)",
1370
+ "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_3.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_3.png</img>\nPrevious Actions: CLICK: (451, 891)",
1371
+ "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_2.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_2.png</img>\nPrevious Actions: CLICK: (451, 891)",
1372
+ "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_1.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_1.png</img>\nPrevious Actions: CLICK: (451, 891)",
1373
+ "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_0.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?"]
1374
+ device = 'cuda'
1375
+
1376
+ batch_raw_text = []
1377
+ for q in qs:
1378
+ raw_text, _ = make_context(tokenizer, q, system="You are a helpful assistant.", max_window_size=6144, chat_format='chatml')
1379
+ batch_raw_text.append(raw_text)
1380
+
1381
+ batch_input = tokenizer(batch_raw_text, return_tensors='pt', padding='longest')
1382
+ batch_input_ids = batch_input['input_ids'].to(device)
1383
+ batch_input_attention_mask = batch_input['attention_mask'].to(device)
1384
+ print(batch_input_ids.shape)
1385
+
1386
+ model = AutoModelForCausalLM.from_pretrained(model_path, device_map=device, trust_remote_code=True).eval()
1387
+ import pdb; pdb.set_trace()
1388
+ batch_out_ids = model.generate(
1389
+ input_ids=batch_input_ids,
1390
+ attention_mask=batch_input_attention_mask,
1391
+ do_sample=False,
1392
+ #top_k=0,
1393
+ #top_p=0.5,
1394
+ num_beams=1,
1395
+ length_penalty=1,
1396
+ num_return_sequences=1,
1397
+ use_cache=True,
1398
+ pad_token_id=tokenizer.eod_id,
1399
+ eos_token_id=tokenizer.eod_id,
1400
+ min_new_tokens=1,
1401
+ max_new_tokens=10,
1402
+ )
1403
+ print(batch_out_ids.shape)
1404
+ print(batch_out_ids)
1405
+ print('-----------------------------------------------')
1406
+
1407
+ padding_lens = [batch_input_ids[i].eq(tokenizer.pad_token_id).sum().item() for i in range(batch_input_ids.size(0))]
1408
+
1409
+ batch_response = [
1410
+ decode_tokens(
1411
+ batch_out_ids[i][padding_lens[i]:],
1412
+ tokenizer,
1413
+ raw_text_len=len(batch_raw_text[i]),
1414
+ context_length=(batch_input_ids[i].size(0)-padding_lens[i]),
1415
+ chat_format="chatml",
1416
+ verbose=False,
1417
+ errors='replace'
1418
+ ) for i in range(len(qs))
1419
+ ]
1420
+ print(batch_input_ids.shape, batch_out_ids.shape)
1421
+ print(batch_response)
pytorch_model-00001-of-00002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:577ac170fa6eefba90aef7a761ae1aa8d9e8fa53c505270ed59b5b34126fd654
3
+ size 9969772092
pytorch_model-00002-of-00002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9a89a7b0c3f3acf02341152c8262b81d90ac457fc2561a6d09c4d2628961e16
3
+ size 9516452100
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,873 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 19485918720
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00002-of-00002.bin",
7
+ "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
8
+ "transformer.h.0.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
9
+ "transformer.h.0.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
10
+ "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
11
+ "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
12
+ "transformer.h.0.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
13
+ "transformer.h.0.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
14
+ "transformer.h.0.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
15
+ "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
16
+ "transformer.h.1.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
17
+ "transformer.h.1.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
18
+ "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
19
+ "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
20
+ "transformer.h.1.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
21
+ "transformer.h.1.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
22
+ "transformer.h.1.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
23
+ "transformer.h.10.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
24
+ "transformer.h.10.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
25
+ "transformer.h.10.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
26
+ "transformer.h.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
27
+ "transformer.h.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
28
+ "transformer.h.10.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
29
+ "transformer.h.10.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
30
+ "transformer.h.10.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
31
+ "transformer.h.11.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
32
+ "transformer.h.11.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
33
+ "transformer.h.11.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
34
+ "transformer.h.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
35
+ "transformer.h.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
36
+ "transformer.h.11.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
37
+ "transformer.h.11.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
38
+ "transformer.h.11.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
39
+ "transformer.h.12.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
40
+ "transformer.h.12.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
41
+ "transformer.h.12.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
42
+ "transformer.h.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
43
+ "transformer.h.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
44
+ "transformer.h.12.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
45
+ "transformer.h.12.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
46
+ "transformer.h.12.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
47
+ "transformer.h.13.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
48
+ "transformer.h.13.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
49
+ "transformer.h.13.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
50
+ "transformer.h.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
51
+ "transformer.h.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
52
+ "transformer.h.13.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
53
+ "transformer.h.13.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
54
+ "transformer.h.13.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
55
+ "transformer.h.14.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
56
+ "transformer.h.14.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
57
+ "transformer.h.14.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
58
+ "transformer.h.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
59
+ "transformer.h.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
60
+ "transformer.h.14.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
61
+ "transformer.h.14.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
62
+ "transformer.h.14.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
63
+ "transformer.h.15.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
64
+ "transformer.h.15.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
65
+ "transformer.h.15.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
66
+ "transformer.h.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
67
+ "transformer.h.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
68
+ "transformer.h.15.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
69
+ "transformer.h.15.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
70
+ "transformer.h.15.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
71
+ "transformer.h.16.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
72
+ "transformer.h.16.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
73
+ "transformer.h.16.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
74
+ "transformer.h.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
75
+ "transformer.h.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
76
+ "transformer.h.16.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
77
+ "transformer.h.16.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
78
+ "transformer.h.16.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
79
+ "transformer.h.17.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
80
+ "transformer.h.17.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
81
+ "transformer.h.17.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
82
+ "transformer.h.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
83
+ "transformer.h.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
84
+ "transformer.h.17.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
85
+ "transformer.h.17.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
86
+ "transformer.h.17.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
87
+ "transformer.h.18.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
88
+ "transformer.h.18.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
89
+ "transformer.h.18.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
90
+ "transformer.h.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
91
+ "transformer.h.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
92
+ "transformer.h.18.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
93
+ "transformer.h.18.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
94
+ "transformer.h.18.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
95
+ "transformer.h.19.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
96
+ "transformer.h.19.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
97
+ "transformer.h.19.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
98
+ "transformer.h.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
99
+ "transformer.h.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
100
+ "transformer.h.19.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
101
+ "transformer.h.19.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
102
+ "transformer.h.19.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
103
+ "transformer.h.2.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
104
+ "transformer.h.2.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
105
+ "transformer.h.2.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
106
+ "transformer.h.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
107
+ "transformer.h.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
108
+ "transformer.h.2.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
109
+ "transformer.h.2.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
110
+ "transformer.h.2.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
111
+ "transformer.h.20.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
112
+ "transformer.h.20.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
113
+ "transformer.h.20.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
114
+ "transformer.h.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
115
+ "transformer.h.20.ln_2.weight": "pytorch_model-00001-of-00002.bin",
116
+ "transformer.h.20.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
117
+ "transformer.h.20.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
118
+ "transformer.h.20.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
119
+ "transformer.h.21.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
120
+ "transformer.h.21.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
121
+ "transformer.h.21.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
122
+ "transformer.h.21.ln_1.weight": "pytorch_model-00001-of-00002.bin",
123
+ "transformer.h.21.ln_2.weight": "pytorch_model-00001-of-00002.bin",
124
+ "transformer.h.21.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
125
+ "transformer.h.21.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
126
+ "transformer.h.21.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
127
+ "transformer.h.22.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
128
+ "transformer.h.22.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
129
+ "transformer.h.22.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
130
+ "transformer.h.22.ln_1.weight": "pytorch_model-00002-of-00002.bin",
131
+ "transformer.h.22.ln_2.weight": "pytorch_model-00002-of-00002.bin",
132
+ "transformer.h.22.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
133
+ "transformer.h.22.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
134
+ "transformer.h.22.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
135
+ "transformer.h.23.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
136
+ "transformer.h.23.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
137
+ "transformer.h.23.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
138
+ "transformer.h.23.ln_1.weight": "pytorch_model-00002-of-00002.bin",
139
+ "transformer.h.23.ln_2.weight": "pytorch_model-00002-of-00002.bin",
140
+ "transformer.h.23.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
141
+ "transformer.h.23.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
142
+ "transformer.h.23.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
143
+ "transformer.h.24.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
144
+ "transformer.h.24.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
145
+ "transformer.h.24.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
146
+ "transformer.h.24.ln_1.weight": "pytorch_model-00002-of-00002.bin",
147
+ "transformer.h.24.ln_2.weight": "pytorch_model-00002-of-00002.bin",
148
+ "transformer.h.24.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
149
+ "transformer.h.24.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
150
+ "transformer.h.24.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
151
+ "transformer.h.25.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
152
+ "transformer.h.25.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
153
+ "transformer.h.25.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
154
+ "transformer.h.25.ln_1.weight": "pytorch_model-00002-of-00002.bin",
155
+ "transformer.h.25.ln_2.weight": "pytorch_model-00002-of-00002.bin",
156
+ "transformer.h.25.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
157
+ "transformer.h.25.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
158
+ "transformer.h.25.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
159
+ "transformer.h.26.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
160
+ "transformer.h.26.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
161
+ "transformer.h.26.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
162
+ "transformer.h.26.ln_1.weight": "pytorch_model-00002-of-00002.bin",
163
+ "transformer.h.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
164
+ "transformer.h.26.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
165
+ "transformer.h.26.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
166
+ "transformer.h.26.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
167
+ "transformer.h.27.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
168
+ "transformer.h.27.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
169
+ "transformer.h.27.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
170
+ "transformer.h.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
171
+ "transformer.h.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
172
+ "transformer.h.27.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
173
+ "transformer.h.27.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
174
+ "transformer.h.27.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
175
+ "transformer.h.28.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
176
+ "transformer.h.28.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
177
+ "transformer.h.28.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
178
+ "transformer.h.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
179
+ "transformer.h.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
180
+ "transformer.h.28.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
181
+ "transformer.h.28.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
182
+ "transformer.h.28.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
183
+ "transformer.h.29.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
184
+ "transformer.h.29.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
185
+ "transformer.h.29.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
186
+ "transformer.h.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
187
+ "transformer.h.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
188
+ "transformer.h.29.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
189
+ "transformer.h.29.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
190
+ "transformer.h.29.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
191
+ "transformer.h.3.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
192
+ "transformer.h.3.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
193
+ "transformer.h.3.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
194
+ "transformer.h.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
195
+ "transformer.h.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
196
+ "transformer.h.3.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
197
+ "transformer.h.3.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
198
+ "transformer.h.3.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
199
+ "transformer.h.30.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
200
+ "transformer.h.30.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
201
+ "transformer.h.30.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
202
+ "transformer.h.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
203
+ "transformer.h.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
204
+ "transformer.h.30.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
205
+ "transformer.h.30.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
206
+ "transformer.h.30.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
207
+ "transformer.h.31.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
208
+ "transformer.h.31.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
209
+ "transformer.h.31.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
210
+ "transformer.h.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
211
+ "transformer.h.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
212
+ "transformer.h.31.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
213
+ "transformer.h.31.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
214
+ "transformer.h.31.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
215
+ "transformer.h.4.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
216
+ "transformer.h.4.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
217
+ "transformer.h.4.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
218
+ "transformer.h.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
219
+ "transformer.h.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
220
+ "transformer.h.4.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
221
+ "transformer.h.4.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
222
+ "transformer.h.4.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
223
+ "transformer.h.5.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
224
+ "transformer.h.5.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
225
+ "transformer.h.5.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
226
+ "transformer.h.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
227
+ "transformer.h.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
228
+ "transformer.h.5.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
229
+ "transformer.h.5.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
230
+ "transformer.h.5.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
231
+ "transformer.h.6.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
232
+ "transformer.h.6.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
233
+ "transformer.h.6.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
234
+ "transformer.h.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
235
+ "transformer.h.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
236
+ "transformer.h.6.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
237
+ "transformer.h.6.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
238
+ "transformer.h.6.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
239
+ "transformer.h.7.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
240
+ "transformer.h.7.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
241
+ "transformer.h.7.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
242
+ "transformer.h.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
243
+ "transformer.h.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
244
+ "transformer.h.7.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
245
+ "transformer.h.7.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
246
+ "transformer.h.7.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
247
+ "transformer.h.8.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
248
+ "transformer.h.8.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
249
+ "transformer.h.8.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
250
+ "transformer.h.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
251
+ "transformer.h.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
252
+ "transformer.h.8.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
253
+ "transformer.h.8.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
254
+ "transformer.h.8.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
255
+ "transformer.h.9.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
256
+ "transformer.h.9.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
257
+ "transformer.h.9.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
258
+ "transformer.h.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
259
+ "transformer.h.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
260
+ "transformer.h.9.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
261
+ "transformer.h.9.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
262
+ "transformer.h.9.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
263
+ "transformer.his_resampler.attn.in_proj_bias": "pytorch_model-00002-of-00002.bin",
264
+ "transformer.his_resampler.attn.in_proj_weight": "pytorch_model-00002-of-00002.bin",
265
+ "transformer.his_resampler.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
266
+ "transformer.his_resampler.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
267
+ "transformer.his_resampler.ln_kv.bias": "pytorch_model-00002-of-00002.bin",
268
+ "transformer.his_resampler.ln_kv.weight": "pytorch_model-00002-of-00002.bin",
269
+ "transformer.his_resampler.ln_post.bias": "pytorch_model-00002-of-00002.bin",
270
+ "transformer.his_resampler.ln_post.weight": "pytorch_model-00002-of-00002.bin",
271
+ "transformer.his_resampler.ln_q.bias": "pytorch_model-00002-of-00002.bin",
272
+ "transformer.his_resampler.ln_q.weight": "pytorch_model-00002-of-00002.bin",
273
+ "transformer.his_resampler.pos_embed": "pytorch_model-00002-of-00002.bin",
274
+ "transformer.his_resampler.proj": "pytorch_model-00002-of-00002.bin",
275
+ "transformer.his_resampler.query": "pytorch_model-00002-of-00002.bin",
276
+ "transformer.ln_f.weight": "pytorch_model-00002-of-00002.bin",
277
+ "transformer.visual.attn_pool.attn.in_proj_bias": "pytorch_model-00002-of-00002.bin",
278
+ "transformer.visual.attn_pool.attn.in_proj_weight": "pytorch_model-00002-of-00002.bin",
279
+ "transformer.visual.attn_pool.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
280
+ "transformer.visual.attn_pool.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
281
+ "transformer.visual.attn_pool.kv_proj.weight": "pytorch_model-00002-of-00002.bin",
282
+ "transformer.visual.attn_pool.ln_kv.bias": "pytorch_model-00002-of-00002.bin",
283
+ "transformer.visual.attn_pool.ln_kv.weight": "pytorch_model-00002-of-00002.bin",
284
+ "transformer.visual.attn_pool.ln_q.bias": "pytorch_model-00002-of-00002.bin",
285
+ "transformer.visual.attn_pool.ln_q.weight": "pytorch_model-00002-of-00002.bin",
286
+ "transformer.visual.attn_pool.pos_embed": "pytorch_model-00002-of-00002.bin",
287
+ "transformer.visual.attn_pool.query": "pytorch_model-00002-of-00002.bin",
288
+ "transformer.visual.conv1.weight": "pytorch_model-00002-of-00002.bin",
289
+ "transformer.visual.ln_post.bias": "pytorch_model-00002-of-00002.bin",
290
+ "transformer.visual.ln_post.weight": "pytorch_model-00002-of-00002.bin",
291
+ "transformer.visual.ln_pre.bias": "pytorch_model-00002-of-00002.bin",
292
+ "transformer.visual.ln_pre.weight": "pytorch_model-00002-of-00002.bin",
293
+ "transformer.visual.positional_embedding": "pytorch_model-00002-of-00002.bin",
294
+ "transformer.visual.proj": "pytorch_model-00002-of-00002.bin",
295
+ "transformer.visual.transformer.resblocks.0.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
296
+ "transformer.visual.transformer.resblocks.0.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
297
+ "transformer.visual.transformer.resblocks.0.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
298
+ "transformer.visual.transformer.resblocks.0.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
299
+ "transformer.visual.transformer.resblocks.0.ln_1.bias": "pytorch_model-00002-of-00002.bin",
300
+ "transformer.visual.transformer.resblocks.0.ln_1.weight": "pytorch_model-00002-of-00002.bin",
301
+ "transformer.visual.transformer.resblocks.0.ln_2.bias": "pytorch_model-00002-of-00002.bin",
302
+ "transformer.visual.transformer.resblocks.0.ln_2.weight": "pytorch_model-00002-of-00002.bin",
303
+ "transformer.visual.transformer.resblocks.0.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
304
+ "transformer.visual.transformer.resblocks.0.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
305
+ "transformer.visual.transformer.resblocks.0.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
306
+ "transformer.visual.transformer.resblocks.0.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
307
+ "transformer.visual.transformer.resblocks.1.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
308
+ "transformer.visual.transformer.resblocks.1.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
309
+ "transformer.visual.transformer.resblocks.1.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
310
+ "transformer.visual.transformer.resblocks.1.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
311
+ "transformer.visual.transformer.resblocks.1.ln_1.bias": "pytorch_model-00002-of-00002.bin",
312
+ "transformer.visual.transformer.resblocks.1.ln_1.weight": "pytorch_model-00002-of-00002.bin",
313
+ "transformer.visual.transformer.resblocks.1.ln_2.bias": "pytorch_model-00002-of-00002.bin",
314
+ "transformer.visual.transformer.resblocks.1.ln_2.weight": "pytorch_model-00002-of-00002.bin",
315
+ "transformer.visual.transformer.resblocks.1.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
316
+ "transformer.visual.transformer.resblocks.1.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
317
+ "transformer.visual.transformer.resblocks.1.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
318
+ "transformer.visual.transformer.resblocks.1.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
319
+ "transformer.visual.transformer.resblocks.10.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
320
+ "transformer.visual.transformer.resblocks.10.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
321
+ "transformer.visual.transformer.resblocks.10.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
322
+ "transformer.visual.transformer.resblocks.10.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
323
+ "transformer.visual.transformer.resblocks.10.ln_1.bias": "pytorch_model-00002-of-00002.bin",
324
+ "transformer.visual.transformer.resblocks.10.ln_1.weight": "pytorch_model-00002-of-00002.bin",
325
+ "transformer.visual.transformer.resblocks.10.ln_2.bias": "pytorch_model-00002-of-00002.bin",
326
+ "transformer.visual.transformer.resblocks.10.ln_2.weight": "pytorch_model-00002-of-00002.bin",
327
+ "transformer.visual.transformer.resblocks.10.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
328
+ "transformer.visual.transformer.resblocks.10.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
329
+ "transformer.visual.transformer.resblocks.10.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
330
+ "transformer.visual.transformer.resblocks.10.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
331
+ "transformer.visual.transformer.resblocks.11.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
332
+ "transformer.visual.transformer.resblocks.11.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
333
+ "transformer.visual.transformer.resblocks.11.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
334
+ "transformer.visual.transformer.resblocks.11.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
335
+ "transformer.visual.transformer.resblocks.11.ln_1.bias": "pytorch_model-00002-of-00002.bin",
336
+ "transformer.visual.transformer.resblocks.11.ln_1.weight": "pytorch_model-00002-of-00002.bin",
337
+ "transformer.visual.transformer.resblocks.11.ln_2.bias": "pytorch_model-00002-of-00002.bin",
338
+ "transformer.visual.transformer.resblocks.11.ln_2.weight": "pytorch_model-00002-of-00002.bin",
339
+ "transformer.visual.transformer.resblocks.11.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
340
+ "transformer.visual.transformer.resblocks.11.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
341
+ "transformer.visual.transformer.resblocks.11.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
342
+ "transformer.visual.transformer.resblocks.11.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
343
+ "transformer.visual.transformer.resblocks.12.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
344
+ "transformer.visual.transformer.resblocks.12.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
345
+ "transformer.visual.transformer.resblocks.12.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
346
+ "transformer.visual.transformer.resblocks.12.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
347
+ "transformer.visual.transformer.resblocks.12.ln_1.bias": "pytorch_model-00002-of-00002.bin",
348
+ "transformer.visual.transformer.resblocks.12.ln_1.weight": "pytorch_model-00002-of-00002.bin",
349
+ "transformer.visual.transformer.resblocks.12.ln_2.bias": "pytorch_model-00002-of-00002.bin",
350
+ "transformer.visual.transformer.resblocks.12.ln_2.weight": "pytorch_model-00002-of-00002.bin",
351
+ "transformer.visual.transformer.resblocks.12.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
352
+ "transformer.visual.transformer.resblocks.12.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
353
+ "transformer.visual.transformer.resblocks.12.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
354
+ "transformer.visual.transformer.resblocks.12.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
355
+ "transformer.visual.transformer.resblocks.13.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
356
+ "transformer.visual.transformer.resblocks.13.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
357
+ "transformer.visual.transformer.resblocks.13.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
358
+ "transformer.visual.transformer.resblocks.13.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
359
+ "transformer.visual.transformer.resblocks.13.ln_1.bias": "pytorch_model-00002-of-00002.bin",
360
+ "transformer.visual.transformer.resblocks.13.ln_1.weight": "pytorch_model-00002-of-00002.bin",
361
+ "transformer.visual.transformer.resblocks.13.ln_2.bias": "pytorch_model-00002-of-00002.bin",
362
+ "transformer.visual.transformer.resblocks.13.ln_2.weight": "pytorch_model-00002-of-00002.bin",
363
+ "transformer.visual.transformer.resblocks.13.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
364
+ "transformer.visual.transformer.resblocks.13.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
365
+ "transformer.visual.transformer.resblocks.13.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
366
+ "transformer.visual.transformer.resblocks.13.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
367
+ "transformer.visual.transformer.resblocks.14.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
368
+ "transformer.visual.transformer.resblocks.14.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
369
+ "transformer.visual.transformer.resblocks.14.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
370
+ "transformer.visual.transformer.resblocks.14.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
371
+ "transformer.visual.transformer.resblocks.14.ln_1.bias": "pytorch_model-00002-of-00002.bin",
372
+ "transformer.visual.transformer.resblocks.14.ln_1.weight": "pytorch_model-00002-of-00002.bin",
373
+ "transformer.visual.transformer.resblocks.14.ln_2.bias": "pytorch_model-00002-of-00002.bin",
374
+ "transformer.visual.transformer.resblocks.14.ln_2.weight": "pytorch_model-00002-of-00002.bin",
375
+ "transformer.visual.transformer.resblocks.14.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
376
+ "transformer.visual.transformer.resblocks.14.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
377
+ "transformer.visual.transformer.resblocks.14.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
378
+ "transformer.visual.transformer.resblocks.14.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
379
+ "transformer.visual.transformer.resblocks.15.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
380
+ "transformer.visual.transformer.resblocks.15.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
381
+ "transformer.visual.transformer.resblocks.15.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
382
+ "transformer.visual.transformer.resblocks.15.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
383
+ "transformer.visual.transformer.resblocks.15.ln_1.bias": "pytorch_model-00002-of-00002.bin",
384
+ "transformer.visual.transformer.resblocks.15.ln_1.weight": "pytorch_model-00002-of-00002.bin",
385
+ "transformer.visual.transformer.resblocks.15.ln_2.bias": "pytorch_model-00002-of-00002.bin",
386
+ "transformer.visual.transformer.resblocks.15.ln_2.weight": "pytorch_model-00002-of-00002.bin",
387
+ "transformer.visual.transformer.resblocks.15.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
388
+ "transformer.visual.transformer.resblocks.15.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
389
+ "transformer.visual.transformer.resblocks.15.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
390
+ "transformer.visual.transformer.resblocks.15.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
391
+ "transformer.visual.transformer.resblocks.16.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
392
+ "transformer.visual.transformer.resblocks.16.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
393
+ "transformer.visual.transformer.resblocks.16.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
394
+ "transformer.visual.transformer.resblocks.16.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
395
+ "transformer.visual.transformer.resblocks.16.ln_1.bias": "pytorch_model-00002-of-00002.bin",
396
+ "transformer.visual.transformer.resblocks.16.ln_1.weight": "pytorch_model-00002-of-00002.bin",
397
+ "transformer.visual.transformer.resblocks.16.ln_2.bias": "pytorch_model-00002-of-00002.bin",
398
+ "transformer.visual.transformer.resblocks.16.ln_2.weight": "pytorch_model-00002-of-00002.bin",
399
+ "transformer.visual.transformer.resblocks.16.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
400
+ "transformer.visual.transformer.resblocks.16.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
401
+ "transformer.visual.transformer.resblocks.16.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
402
+ "transformer.visual.transformer.resblocks.16.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
403
+ "transformer.visual.transformer.resblocks.17.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
404
+ "transformer.visual.transformer.resblocks.17.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
405
+ "transformer.visual.transformer.resblocks.17.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
406
+ "transformer.visual.transformer.resblocks.17.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
407
+ "transformer.visual.transformer.resblocks.17.ln_1.bias": "pytorch_model-00002-of-00002.bin",
408
+ "transformer.visual.transformer.resblocks.17.ln_1.weight": "pytorch_model-00002-of-00002.bin",
409
+ "transformer.visual.transformer.resblocks.17.ln_2.bias": "pytorch_model-00002-of-00002.bin",
410
+ "transformer.visual.transformer.resblocks.17.ln_2.weight": "pytorch_model-00002-of-00002.bin",
411
+ "transformer.visual.transformer.resblocks.17.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
412
+ "transformer.visual.transformer.resblocks.17.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
413
+ "transformer.visual.transformer.resblocks.17.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
414
+ "transformer.visual.transformer.resblocks.17.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
415
+ "transformer.visual.transformer.resblocks.18.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
416
+ "transformer.visual.transformer.resblocks.18.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
417
+ "transformer.visual.transformer.resblocks.18.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
418
+ "transformer.visual.transformer.resblocks.18.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
419
+ "transformer.visual.transformer.resblocks.18.ln_1.bias": "pytorch_model-00002-of-00002.bin",
420
+ "transformer.visual.transformer.resblocks.18.ln_1.weight": "pytorch_model-00002-of-00002.bin",
421
+ "transformer.visual.transformer.resblocks.18.ln_2.bias": "pytorch_model-00002-of-00002.bin",
422
+ "transformer.visual.transformer.resblocks.18.ln_2.weight": "pytorch_model-00002-of-00002.bin",
423
+ "transformer.visual.transformer.resblocks.18.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
424
+ "transformer.visual.transformer.resblocks.18.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
425
+ "transformer.visual.transformer.resblocks.18.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
426
+ "transformer.visual.transformer.resblocks.18.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
427
+ "transformer.visual.transformer.resblocks.19.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
428
+ "transformer.visual.transformer.resblocks.19.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
429
+ "transformer.visual.transformer.resblocks.19.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
430
+ "transformer.visual.transformer.resblocks.19.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
431
+ "transformer.visual.transformer.resblocks.19.ln_1.bias": "pytorch_model-00002-of-00002.bin",
432
+ "transformer.visual.transformer.resblocks.19.ln_1.weight": "pytorch_model-00002-of-00002.bin",
433
+ "transformer.visual.transformer.resblocks.19.ln_2.bias": "pytorch_model-00002-of-00002.bin",
434
+ "transformer.visual.transformer.resblocks.19.ln_2.weight": "pytorch_model-00002-of-00002.bin",
435
+ "transformer.visual.transformer.resblocks.19.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
436
+ "transformer.visual.transformer.resblocks.19.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
437
+ "transformer.visual.transformer.resblocks.19.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
438
+ "transformer.visual.transformer.resblocks.19.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
439
+ "transformer.visual.transformer.resblocks.2.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
440
+ "transformer.visual.transformer.resblocks.2.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
441
+ "transformer.visual.transformer.resblocks.2.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
442
+ "transformer.visual.transformer.resblocks.2.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
443
+ "transformer.visual.transformer.resblocks.2.ln_1.bias": "pytorch_model-00002-of-00002.bin",
444
+ "transformer.visual.transformer.resblocks.2.ln_1.weight": "pytorch_model-00002-of-00002.bin",
445
+ "transformer.visual.transformer.resblocks.2.ln_2.bias": "pytorch_model-00002-of-00002.bin",
446
+ "transformer.visual.transformer.resblocks.2.ln_2.weight": "pytorch_model-00002-of-00002.bin",
447
+ "transformer.visual.transformer.resblocks.2.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
448
+ "transformer.visual.transformer.resblocks.2.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
449
+ "transformer.visual.transformer.resblocks.2.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
450
+ "transformer.visual.transformer.resblocks.2.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
451
+ "transformer.visual.transformer.resblocks.20.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
452
+ "transformer.visual.transformer.resblocks.20.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
453
+ "transformer.visual.transformer.resblocks.20.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
454
+ "transformer.visual.transformer.resblocks.20.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
455
+ "transformer.visual.transformer.resblocks.20.ln_1.bias": "pytorch_model-00002-of-00002.bin",
456
+ "transformer.visual.transformer.resblocks.20.ln_1.weight": "pytorch_model-00002-of-00002.bin",
457
+ "transformer.visual.transformer.resblocks.20.ln_2.bias": "pytorch_model-00002-of-00002.bin",
458
+ "transformer.visual.transformer.resblocks.20.ln_2.weight": "pytorch_model-00002-of-00002.bin",
459
+ "transformer.visual.transformer.resblocks.20.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
460
+ "transformer.visual.transformer.resblocks.20.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
461
+ "transformer.visual.transformer.resblocks.20.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
462
+ "transformer.visual.transformer.resblocks.20.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
463
+ "transformer.visual.transformer.resblocks.21.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
464
+ "transformer.visual.transformer.resblocks.21.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
465
+ "transformer.visual.transformer.resblocks.21.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
466
+ "transformer.visual.transformer.resblocks.21.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
467
+ "transformer.visual.transformer.resblocks.21.ln_1.bias": "pytorch_model-00002-of-00002.bin",
468
+ "transformer.visual.transformer.resblocks.21.ln_1.weight": "pytorch_model-00002-of-00002.bin",
469
+ "transformer.visual.transformer.resblocks.21.ln_2.bias": "pytorch_model-00002-of-00002.bin",
470
+ "transformer.visual.transformer.resblocks.21.ln_2.weight": "pytorch_model-00002-of-00002.bin",
471
+ "transformer.visual.transformer.resblocks.21.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
472
+ "transformer.visual.transformer.resblocks.21.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
473
+ "transformer.visual.transformer.resblocks.21.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
474
+ "transformer.visual.transformer.resblocks.21.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
475
+ "transformer.visual.transformer.resblocks.22.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
476
+ "transformer.visual.transformer.resblocks.22.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
477
+ "transformer.visual.transformer.resblocks.22.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
478
+ "transformer.visual.transformer.resblocks.22.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
479
+ "transformer.visual.transformer.resblocks.22.ln_1.bias": "pytorch_model-00002-of-00002.bin",
480
+ "transformer.visual.transformer.resblocks.22.ln_1.weight": "pytorch_model-00002-of-00002.bin",
481
+ "transformer.visual.transformer.resblocks.22.ln_2.bias": "pytorch_model-00002-of-00002.bin",
482
+ "transformer.visual.transformer.resblocks.22.ln_2.weight": "pytorch_model-00002-of-00002.bin",
483
+ "transformer.visual.transformer.resblocks.22.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
484
+ "transformer.visual.transformer.resblocks.22.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
485
+ "transformer.visual.transformer.resblocks.22.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
486
+ "transformer.visual.transformer.resblocks.22.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
487
+ "transformer.visual.transformer.resblocks.23.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
488
+ "transformer.visual.transformer.resblocks.23.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
489
+ "transformer.visual.transformer.resblocks.23.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
490
+ "transformer.visual.transformer.resblocks.23.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
491
+ "transformer.visual.transformer.resblocks.23.ln_1.bias": "pytorch_model-00002-of-00002.bin",
492
+ "transformer.visual.transformer.resblocks.23.ln_1.weight": "pytorch_model-00002-of-00002.bin",
493
+ "transformer.visual.transformer.resblocks.23.ln_2.bias": "pytorch_model-00002-of-00002.bin",
494
+ "transformer.visual.transformer.resblocks.23.ln_2.weight": "pytorch_model-00002-of-00002.bin",
495
+ "transformer.visual.transformer.resblocks.23.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
496
+ "transformer.visual.transformer.resblocks.23.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
497
+ "transformer.visual.transformer.resblocks.23.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
498
+ "transformer.visual.transformer.resblocks.23.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
499
+ "transformer.visual.transformer.resblocks.24.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
500
+ "transformer.visual.transformer.resblocks.24.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
501
+ "transformer.visual.transformer.resblocks.24.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
502
+ "transformer.visual.transformer.resblocks.24.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
503
+ "transformer.visual.transformer.resblocks.24.ln_1.bias": "pytorch_model-00002-of-00002.bin",
504
+ "transformer.visual.transformer.resblocks.24.ln_1.weight": "pytorch_model-00002-of-00002.bin",
505
+ "transformer.visual.transformer.resblocks.24.ln_2.bias": "pytorch_model-00002-of-00002.bin",
506
+ "transformer.visual.transformer.resblocks.24.ln_2.weight": "pytorch_model-00002-of-00002.bin",
507
+ "transformer.visual.transformer.resblocks.24.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
508
+ "transformer.visual.transformer.resblocks.24.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
509
+ "transformer.visual.transformer.resblocks.24.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
510
+ "transformer.visual.transformer.resblocks.24.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
511
+ "transformer.visual.transformer.resblocks.25.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
512
+ "transformer.visual.transformer.resblocks.25.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
513
+ "transformer.visual.transformer.resblocks.25.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
514
+ "transformer.visual.transformer.resblocks.25.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
515
+ "transformer.visual.transformer.resblocks.25.ln_1.bias": "pytorch_model-00002-of-00002.bin",
516
+ "transformer.visual.transformer.resblocks.25.ln_1.weight": "pytorch_model-00002-of-00002.bin",
517
+ "transformer.visual.transformer.resblocks.25.ln_2.bias": "pytorch_model-00002-of-00002.bin",
518
+ "transformer.visual.transformer.resblocks.25.ln_2.weight": "pytorch_model-00002-of-00002.bin",
519
+ "transformer.visual.transformer.resblocks.25.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
520
+ "transformer.visual.transformer.resblocks.25.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
521
+ "transformer.visual.transformer.resblocks.25.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
522
+ "transformer.visual.transformer.resblocks.25.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
523
+ "transformer.visual.transformer.resblocks.26.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
524
+ "transformer.visual.transformer.resblocks.26.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
525
+ "transformer.visual.transformer.resblocks.26.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
526
+ "transformer.visual.transformer.resblocks.26.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
527
+ "transformer.visual.transformer.resblocks.26.ln_1.bias": "pytorch_model-00002-of-00002.bin",
528
+ "transformer.visual.transformer.resblocks.26.ln_1.weight": "pytorch_model-00002-of-00002.bin",
529
+ "transformer.visual.transformer.resblocks.26.ln_2.bias": "pytorch_model-00002-of-00002.bin",
530
+ "transformer.visual.transformer.resblocks.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
531
+ "transformer.visual.transformer.resblocks.26.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
532
+ "transformer.visual.transformer.resblocks.26.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
533
+ "transformer.visual.transformer.resblocks.26.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
534
+ "transformer.visual.transformer.resblocks.26.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
535
+ "transformer.visual.transformer.resblocks.27.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
536
+ "transformer.visual.transformer.resblocks.27.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
537
+ "transformer.visual.transformer.resblocks.27.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
538
+ "transformer.visual.transformer.resblocks.27.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
539
+ "transformer.visual.transformer.resblocks.27.ln_1.bias": "pytorch_model-00002-of-00002.bin",
540
+ "transformer.visual.transformer.resblocks.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
541
+ "transformer.visual.transformer.resblocks.27.ln_2.bias": "pytorch_model-00002-of-00002.bin",
542
+ "transformer.visual.transformer.resblocks.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
543
+ "transformer.visual.transformer.resblocks.27.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
544
+ "transformer.visual.transformer.resblocks.27.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
545
+ "transformer.visual.transformer.resblocks.27.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
546
+ "transformer.visual.transformer.resblocks.27.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
547
+ "transformer.visual.transformer.resblocks.28.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
548
+ "transformer.visual.transformer.resblocks.28.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
549
+ "transformer.visual.transformer.resblocks.28.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
550
+ "transformer.visual.transformer.resblocks.28.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
551
+ "transformer.visual.transformer.resblocks.28.ln_1.bias": "pytorch_model-00002-of-00002.bin",
552
+ "transformer.visual.transformer.resblocks.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
553
+ "transformer.visual.transformer.resblocks.28.ln_2.bias": "pytorch_model-00002-of-00002.bin",
554
+ "transformer.visual.transformer.resblocks.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
555
+ "transformer.visual.transformer.resblocks.28.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
556
+ "transformer.visual.transformer.resblocks.28.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
557
+ "transformer.visual.transformer.resblocks.28.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
558
+ "transformer.visual.transformer.resblocks.28.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
559
+ "transformer.visual.transformer.resblocks.29.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
560
+ "transformer.visual.transformer.resblocks.29.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
561
+ "transformer.visual.transformer.resblocks.29.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
562
+ "transformer.visual.transformer.resblocks.29.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
563
+ "transformer.visual.transformer.resblocks.29.ln_1.bias": "pytorch_model-00002-of-00002.bin",
564
+ "transformer.visual.transformer.resblocks.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
565
+ "transformer.visual.transformer.resblocks.29.ln_2.bias": "pytorch_model-00002-of-00002.bin",
566
+ "transformer.visual.transformer.resblocks.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
567
+ "transformer.visual.transformer.resblocks.29.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
568
+ "transformer.visual.transformer.resblocks.29.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
569
+ "transformer.visual.transformer.resblocks.29.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
570
+ "transformer.visual.transformer.resblocks.29.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
571
+ "transformer.visual.transformer.resblocks.3.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
572
+ "transformer.visual.transformer.resblocks.3.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
573
+ "transformer.visual.transformer.resblocks.3.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
574
+ "transformer.visual.transformer.resblocks.3.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
575
+ "transformer.visual.transformer.resblocks.3.ln_1.bias": "pytorch_model-00002-of-00002.bin",
576
+ "transformer.visual.transformer.resblocks.3.ln_1.weight": "pytorch_model-00002-of-00002.bin",
577
+ "transformer.visual.transformer.resblocks.3.ln_2.bias": "pytorch_model-00002-of-00002.bin",
578
+ "transformer.visual.transformer.resblocks.3.ln_2.weight": "pytorch_model-00002-of-00002.bin",
579
+ "transformer.visual.transformer.resblocks.3.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
580
+ "transformer.visual.transformer.resblocks.3.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
581
+ "transformer.visual.transformer.resblocks.3.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
582
+ "transformer.visual.transformer.resblocks.3.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
583
+ "transformer.visual.transformer.resblocks.30.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
584
+ "transformer.visual.transformer.resblocks.30.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
585
+ "transformer.visual.transformer.resblocks.30.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
586
+ "transformer.visual.transformer.resblocks.30.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
587
+ "transformer.visual.transformer.resblocks.30.ln_1.bias": "pytorch_model-00002-of-00002.bin",
588
+ "transformer.visual.transformer.resblocks.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
589
+ "transformer.visual.transformer.resblocks.30.ln_2.bias": "pytorch_model-00002-of-00002.bin",
590
+ "transformer.visual.transformer.resblocks.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
591
+ "transformer.visual.transformer.resblocks.30.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
592
+ "transformer.visual.transformer.resblocks.30.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
593
+ "transformer.visual.transformer.resblocks.30.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
594
+ "transformer.visual.transformer.resblocks.30.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
595
+ "transformer.visual.transformer.resblocks.31.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
596
+ "transformer.visual.transformer.resblocks.31.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
597
+ "transformer.visual.transformer.resblocks.31.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
598
+ "transformer.visual.transformer.resblocks.31.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
599
+ "transformer.visual.transformer.resblocks.31.ln_1.bias": "pytorch_model-00002-of-00002.bin",
600
+ "transformer.visual.transformer.resblocks.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
601
+ "transformer.visual.transformer.resblocks.31.ln_2.bias": "pytorch_model-00002-of-00002.bin",
602
+ "transformer.visual.transformer.resblocks.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
603
+ "transformer.visual.transformer.resblocks.31.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
604
+ "transformer.visual.transformer.resblocks.31.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
605
+ "transformer.visual.transformer.resblocks.31.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
606
+ "transformer.visual.transformer.resblocks.31.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
607
+ "transformer.visual.transformer.resblocks.32.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
608
+ "transformer.visual.transformer.resblocks.32.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
609
+ "transformer.visual.transformer.resblocks.32.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
610
+ "transformer.visual.transformer.resblocks.32.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
611
+ "transformer.visual.transformer.resblocks.32.ln_1.bias": "pytorch_model-00002-of-00002.bin",
612
+ "transformer.visual.transformer.resblocks.32.ln_1.weight": "pytorch_model-00002-of-00002.bin",
613
+ "transformer.visual.transformer.resblocks.32.ln_2.bias": "pytorch_model-00002-of-00002.bin",
614
+ "transformer.visual.transformer.resblocks.32.ln_2.weight": "pytorch_model-00002-of-00002.bin",
615
+ "transformer.visual.transformer.resblocks.32.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
616
+ "transformer.visual.transformer.resblocks.32.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
617
+ "transformer.visual.transformer.resblocks.32.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
618
+ "transformer.visual.transformer.resblocks.32.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
619
+ "transformer.visual.transformer.resblocks.33.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
620
+ "transformer.visual.transformer.resblocks.33.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
621
+ "transformer.visual.transformer.resblocks.33.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
622
+ "transformer.visual.transformer.resblocks.33.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
623
+ "transformer.visual.transformer.resblocks.33.ln_1.bias": "pytorch_model-00002-of-00002.bin",
624
+ "transformer.visual.transformer.resblocks.33.ln_1.weight": "pytorch_model-00002-of-00002.bin",
625
+ "transformer.visual.transformer.resblocks.33.ln_2.bias": "pytorch_model-00002-of-00002.bin",
626
+ "transformer.visual.transformer.resblocks.33.ln_2.weight": "pytorch_model-00002-of-00002.bin",
627
+ "transformer.visual.transformer.resblocks.33.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
628
+ "transformer.visual.transformer.resblocks.33.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
629
+ "transformer.visual.transformer.resblocks.33.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
630
+ "transformer.visual.transformer.resblocks.33.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
631
+ "transformer.visual.transformer.resblocks.34.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
632
+ "transformer.visual.transformer.resblocks.34.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
633
+ "transformer.visual.transformer.resblocks.34.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
634
+ "transformer.visual.transformer.resblocks.34.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
635
+ "transformer.visual.transformer.resblocks.34.ln_1.bias": "pytorch_model-00002-of-00002.bin",
636
+ "transformer.visual.transformer.resblocks.34.ln_1.weight": "pytorch_model-00002-of-00002.bin",
637
+ "transformer.visual.transformer.resblocks.34.ln_2.bias": "pytorch_model-00002-of-00002.bin",
638
+ "transformer.visual.transformer.resblocks.34.ln_2.weight": "pytorch_model-00002-of-00002.bin",
639
+ "transformer.visual.transformer.resblocks.34.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
640
+ "transformer.visual.transformer.resblocks.34.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
641
+ "transformer.visual.transformer.resblocks.34.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
642
+ "transformer.visual.transformer.resblocks.34.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
643
+ "transformer.visual.transformer.resblocks.35.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
644
+ "transformer.visual.transformer.resblocks.35.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
645
+ "transformer.visual.transformer.resblocks.35.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
646
+ "transformer.visual.transformer.resblocks.35.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
647
+ "transformer.visual.transformer.resblocks.35.ln_1.bias": "pytorch_model-00002-of-00002.bin",
648
+ "transformer.visual.transformer.resblocks.35.ln_1.weight": "pytorch_model-00002-of-00002.bin",
649
+ "transformer.visual.transformer.resblocks.35.ln_2.bias": "pytorch_model-00002-of-00002.bin",
650
+ "transformer.visual.transformer.resblocks.35.ln_2.weight": "pytorch_model-00002-of-00002.bin",
651
+ "transformer.visual.transformer.resblocks.35.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
652
+ "transformer.visual.transformer.resblocks.35.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
653
+ "transformer.visual.transformer.resblocks.35.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
654
+ "transformer.visual.transformer.resblocks.35.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
655
+ "transformer.visual.transformer.resblocks.36.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
656
+ "transformer.visual.transformer.resblocks.36.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
657
+ "transformer.visual.transformer.resblocks.36.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
658
+ "transformer.visual.transformer.resblocks.36.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
659
+ "transformer.visual.transformer.resblocks.36.ln_1.bias": "pytorch_model-00002-of-00002.bin",
660
+ "transformer.visual.transformer.resblocks.36.ln_1.weight": "pytorch_model-00002-of-00002.bin",
661
+ "transformer.visual.transformer.resblocks.36.ln_2.bias": "pytorch_model-00002-of-00002.bin",
662
+ "transformer.visual.transformer.resblocks.36.ln_2.weight": "pytorch_model-00002-of-00002.bin",
663
+ "transformer.visual.transformer.resblocks.36.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
664
+ "transformer.visual.transformer.resblocks.36.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
665
+ "transformer.visual.transformer.resblocks.36.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
666
+ "transformer.visual.transformer.resblocks.36.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
667
+ "transformer.visual.transformer.resblocks.37.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
668
+ "transformer.visual.transformer.resblocks.37.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
669
+ "transformer.visual.transformer.resblocks.37.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
670
+ "transformer.visual.transformer.resblocks.37.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
671
+ "transformer.visual.transformer.resblocks.37.ln_1.bias": "pytorch_model-00002-of-00002.bin",
672
+ "transformer.visual.transformer.resblocks.37.ln_1.weight": "pytorch_model-00002-of-00002.bin",
673
+ "transformer.visual.transformer.resblocks.37.ln_2.bias": "pytorch_model-00002-of-00002.bin",
674
+ "transformer.visual.transformer.resblocks.37.ln_2.weight": "pytorch_model-00002-of-00002.bin",
675
+ "transformer.visual.transformer.resblocks.37.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
676
+ "transformer.visual.transformer.resblocks.37.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
677
+ "transformer.visual.transformer.resblocks.37.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
678
+ "transformer.visual.transformer.resblocks.37.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
679
+ "transformer.visual.transformer.resblocks.38.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
680
+ "transformer.visual.transformer.resblocks.38.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
681
+ "transformer.visual.transformer.resblocks.38.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
682
+ "transformer.visual.transformer.resblocks.38.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
683
+ "transformer.visual.transformer.resblocks.38.ln_1.bias": "pytorch_model-00002-of-00002.bin",
684
+ "transformer.visual.transformer.resblocks.38.ln_1.weight": "pytorch_model-00002-of-00002.bin",
685
+ "transformer.visual.transformer.resblocks.38.ln_2.bias": "pytorch_model-00002-of-00002.bin",
686
+ "transformer.visual.transformer.resblocks.38.ln_2.weight": "pytorch_model-00002-of-00002.bin",
687
+ "transformer.visual.transformer.resblocks.38.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
688
+ "transformer.visual.transformer.resblocks.38.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
689
+ "transformer.visual.transformer.resblocks.38.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
690
+ "transformer.visual.transformer.resblocks.38.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
691
+ "transformer.visual.transformer.resblocks.39.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
692
+ "transformer.visual.transformer.resblocks.39.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
693
+ "transformer.visual.transformer.resblocks.39.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
694
+ "transformer.visual.transformer.resblocks.39.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
695
+ "transformer.visual.transformer.resblocks.39.ln_1.bias": "pytorch_model-00002-of-00002.bin",
696
+ "transformer.visual.transformer.resblocks.39.ln_1.weight": "pytorch_model-00002-of-00002.bin",
697
+ "transformer.visual.transformer.resblocks.39.ln_2.bias": "pytorch_model-00002-of-00002.bin",
698
+ "transformer.visual.transformer.resblocks.39.ln_2.weight": "pytorch_model-00002-of-00002.bin",
699
+ "transformer.visual.transformer.resblocks.39.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
700
+ "transformer.visual.transformer.resblocks.39.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
701
+ "transformer.visual.transformer.resblocks.39.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
702
+ "transformer.visual.transformer.resblocks.39.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
703
+ "transformer.visual.transformer.resblocks.4.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
704
+ "transformer.visual.transformer.resblocks.4.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
705
+ "transformer.visual.transformer.resblocks.4.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
706
+ "transformer.visual.transformer.resblocks.4.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
707
+ "transformer.visual.transformer.resblocks.4.ln_1.bias": "pytorch_model-00002-of-00002.bin",
708
+ "transformer.visual.transformer.resblocks.4.ln_1.weight": "pytorch_model-00002-of-00002.bin",
709
+ "transformer.visual.transformer.resblocks.4.ln_2.bias": "pytorch_model-00002-of-00002.bin",
710
+ "transformer.visual.transformer.resblocks.4.ln_2.weight": "pytorch_model-00002-of-00002.bin",
711
+ "transformer.visual.transformer.resblocks.4.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
712
+ "transformer.visual.transformer.resblocks.4.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
713
+ "transformer.visual.transformer.resblocks.4.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
714
+ "transformer.visual.transformer.resblocks.4.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
715
+ "transformer.visual.transformer.resblocks.40.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
716
+ "transformer.visual.transformer.resblocks.40.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
717
+ "transformer.visual.transformer.resblocks.40.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
718
+ "transformer.visual.transformer.resblocks.40.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
719
+ "transformer.visual.transformer.resblocks.40.ln_1.bias": "pytorch_model-00002-of-00002.bin",
720
+ "transformer.visual.transformer.resblocks.40.ln_1.weight": "pytorch_model-00002-of-00002.bin",
721
+ "transformer.visual.transformer.resblocks.40.ln_2.bias": "pytorch_model-00002-of-00002.bin",
722
+ "transformer.visual.transformer.resblocks.40.ln_2.weight": "pytorch_model-00002-of-00002.bin",
723
+ "transformer.visual.transformer.resblocks.40.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
724
+ "transformer.visual.transformer.resblocks.40.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
725
+ "transformer.visual.transformer.resblocks.40.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
726
+ "transformer.visual.transformer.resblocks.40.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
727
+ "transformer.visual.transformer.resblocks.41.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
728
+ "transformer.visual.transformer.resblocks.41.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
729
+ "transformer.visual.transformer.resblocks.41.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
730
+ "transformer.visual.transformer.resblocks.41.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
731
+ "transformer.visual.transformer.resblocks.41.ln_1.bias": "pytorch_model-00002-of-00002.bin",
732
+ "transformer.visual.transformer.resblocks.41.ln_1.weight": "pytorch_model-00002-of-00002.bin",
733
+ "transformer.visual.transformer.resblocks.41.ln_2.bias": "pytorch_model-00002-of-00002.bin",
734
+ "transformer.visual.transformer.resblocks.41.ln_2.weight": "pytorch_model-00002-of-00002.bin",
735
+ "transformer.visual.transformer.resblocks.41.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
736
+ "transformer.visual.transformer.resblocks.41.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
737
+ "transformer.visual.transformer.resblocks.41.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
738
+ "transformer.visual.transformer.resblocks.41.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
739
+ "transformer.visual.transformer.resblocks.42.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
740
+ "transformer.visual.transformer.resblocks.42.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
741
+ "transformer.visual.transformer.resblocks.42.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
742
+ "transformer.visual.transformer.resblocks.42.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
743
+ "transformer.visual.transformer.resblocks.42.ln_1.bias": "pytorch_model-00002-of-00002.bin",
744
+ "transformer.visual.transformer.resblocks.42.ln_1.weight": "pytorch_model-00002-of-00002.bin",
745
+ "transformer.visual.transformer.resblocks.42.ln_2.bias": "pytorch_model-00002-of-00002.bin",
746
+ "transformer.visual.transformer.resblocks.42.ln_2.weight": "pytorch_model-00002-of-00002.bin",
747
+ "transformer.visual.transformer.resblocks.42.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
748
+ "transformer.visual.transformer.resblocks.42.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
749
+ "transformer.visual.transformer.resblocks.42.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
750
+ "transformer.visual.transformer.resblocks.42.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
751
+ "transformer.visual.transformer.resblocks.43.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
752
+ "transformer.visual.transformer.resblocks.43.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
753
+ "transformer.visual.transformer.resblocks.43.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
754
+ "transformer.visual.transformer.resblocks.43.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
755
+ "transformer.visual.transformer.resblocks.43.ln_1.bias": "pytorch_model-00002-of-00002.bin",
756
+ "transformer.visual.transformer.resblocks.43.ln_1.weight": "pytorch_model-00002-of-00002.bin",
757
+ "transformer.visual.transformer.resblocks.43.ln_2.bias": "pytorch_model-00002-of-00002.bin",
758
+ "transformer.visual.transformer.resblocks.43.ln_2.weight": "pytorch_model-00002-of-00002.bin",
759
+ "transformer.visual.transformer.resblocks.43.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
760
+ "transformer.visual.transformer.resblocks.43.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
761
+ "transformer.visual.transformer.resblocks.43.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
762
+ "transformer.visual.transformer.resblocks.43.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
763
+ "transformer.visual.transformer.resblocks.44.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
764
+ "transformer.visual.transformer.resblocks.44.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
765
+ "transformer.visual.transformer.resblocks.44.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
766
+ "transformer.visual.transformer.resblocks.44.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
767
+ "transformer.visual.transformer.resblocks.44.ln_1.bias": "pytorch_model-00002-of-00002.bin",
768
+ "transformer.visual.transformer.resblocks.44.ln_1.weight": "pytorch_model-00002-of-00002.bin",
769
+ "transformer.visual.transformer.resblocks.44.ln_2.bias": "pytorch_model-00002-of-00002.bin",
770
+ "transformer.visual.transformer.resblocks.44.ln_2.weight": "pytorch_model-00002-of-00002.bin",
771
+ "transformer.visual.transformer.resblocks.44.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
772
+ "transformer.visual.transformer.resblocks.44.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
773
+ "transformer.visual.transformer.resblocks.44.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
774
+ "transformer.visual.transformer.resblocks.44.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
775
+ "transformer.visual.transformer.resblocks.45.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
776
+ "transformer.visual.transformer.resblocks.45.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
777
+ "transformer.visual.transformer.resblocks.45.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
778
+ "transformer.visual.transformer.resblocks.45.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
779
+ "transformer.visual.transformer.resblocks.45.ln_1.bias": "pytorch_model-00002-of-00002.bin",
780
+ "transformer.visual.transformer.resblocks.45.ln_1.weight": "pytorch_model-00002-of-00002.bin",
781
+ "transformer.visual.transformer.resblocks.45.ln_2.bias": "pytorch_model-00002-of-00002.bin",
782
+ "transformer.visual.transformer.resblocks.45.ln_2.weight": "pytorch_model-00002-of-00002.bin",
783
+ "transformer.visual.transformer.resblocks.45.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
784
+ "transformer.visual.transformer.resblocks.45.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
785
+ "transformer.visual.transformer.resblocks.45.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
786
+ "transformer.visual.transformer.resblocks.45.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
787
+ "transformer.visual.transformer.resblocks.46.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
788
+ "transformer.visual.transformer.resblocks.46.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
789
+ "transformer.visual.transformer.resblocks.46.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
790
+ "transformer.visual.transformer.resblocks.46.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
791
+ "transformer.visual.transformer.resblocks.46.ln_1.bias": "pytorch_model-00002-of-00002.bin",
792
+ "transformer.visual.transformer.resblocks.46.ln_1.weight": "pytorch_model-00002-of-00002.bin",
793
+ "transformer.visual.transformer.resblocks.46.ln_2.bias": "pytorch_model-00002-of-00002.bin",
794
+ "transformer.visual.transformer.resblocks.46.ln_2.weight": "pytorch_model-00002-of-00002.bin",
795
+ "transformer.visual.transformer.resblocks.46.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
796
+ "transformer.visual.transformer.resblocks.46.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
797
+ "transformer.visual.transformer.resblocks.46.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
798
+ "transformer.visual.transformer.resblocks.46.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
799
+ "transformer.visual.transformer.resblocks.47.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
800
+ "transformer.visual.transformer.resblocks.47.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
801
+ "transformer.visual.transformer.resblocks.47.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
802
+ "transformer.visual.transformer.resblocks.47.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
803
+ "transformer.visual.transformer.resblocks.47.ln_1.bias": "pytorch_model-00002-of-00002.bin",
804
+ "transformer.visual.transformer.resblocks.47.ln_1.weight": "pytorch_model-00002-of-00002.bin",
805
+ "transformer.visual.transformer.resblocks.47.ln_2.bias": "pytorch_model-00002-of-00002.bin",
806
+ "transformer.visual.transformer.resblocks.47.ln_2.weight": "pytorch_model-00002-of-00002.bin",
807
+ "transformer.visual.transformer.resblocks.47.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
808
+ "transformer.visual.transformer.resblocks.47.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
809
+ "transformer.visual.transformer.resblocks.47.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
810
+ "transformer.visual.transformer.resblocks.47.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
811
+ "transformer.visual.transformer.resblocks.5.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
812
+ "transformer.visual.transformer.resblocks.5.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
813
+ "transformer.visual.transformer.resblocks.5.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
814
+ "transformer.visual.transformer.resblocks.5.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
815
+ "transformer.visual.transformer.resblocks.5.ln_1.bias": "pytorch_model-00002-of-00002.bin",
816
+ "transformer.visual.transformer.resblocks.5.ln_1.weight": "pytorch_model-00002-of-00002.bin",
817
+ "transformer.visual.transformer.resblocks.5.ln_2.bias": "pytorch_model-00002-of-00002.bin",
818
+ "transformer.visual.transformer.resblocks.5.ln_2.weight": "pytorch_model-00002-of-00002.bin",
819
+ "transformer.visual.transformer.resblocks.5.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
820
+ "transformer.visual.transformer.resblocks.5.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
821
+ "transformer.visual.transformer.resblocks.5.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
822
+ "transformer.visual.transformer.resblocks.5.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
823
+ "transformer.visual.transformer.resblocks.6.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
824
+ "transformer.visual.transformer.resblocks.6.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
825
+ "transformer.visual.transformer.resblocks.6.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
826
+ "transformer.visual.transformer.resblocks.6.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
827
+ "transformer.visual.transformer.resblocks.6.ln_1.bias": "pytorch_model-00002-of-00002.bin",
828
+ "transformer.visual.transformer.resblocks.6.ln_1.weight": "pytorch_model-00002-of-00002.bin",
829
+ "transformer.visual.transformer.resblocks.6.ln_2.bias": "pytorch_model-00002-of-00002.bin",
830
+ "transformer.visual.transformer.resblocks.6.ln_2.weight": "pytorch_model-00002-of-00002.bin",
831
+ "transformer.visual.transformer.resblocks.6.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
832
+ "transformer.visual.transformer.resblocks.6.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
833
+ "transformer.visual.transformer.resblocks.6.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
834
+ "transformer.visual.transformer.resblocks.6.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
835
+ "transformer.visual.transformer.resblocks.7.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
836
+ "transformer.visual.transformer.resblocks.7.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
837
+ "transformer.visual.transformer.resblocks.7.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
838
+ "transformer.visual.transformer.resblocks.7.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
839
+ "transformer.visual.transformer.resblocks.7.ln_1.bias": "pytorch_model-00002-of-00002.bin",
840
+ "transformer.visual.transformer.resblocks.7.ln_1.weight": "pytorch_model-00002-of-00002.bin",
841
+ "transformer.visual.transformer.resblocks.7.ln_2.bias": "pytorch_model-00002-of-00002.bin",
842
+ "transformer.visual.transformer.resblocks.7.ln_2.weight": "pytorch_model-00002-of-00002.bin",
843
+ "transformer.visual.transformer.resblocks.7.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
844
+ "transformer.visual.transformer.resblocks.7.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
845
+ "transformer.visual.transformer.resblocks.7.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
846
+ "transformer.visual.transformer.resblocks.7.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
847
+ "transformer.visual.transformer.resblocks.8.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
848
+ "transformer.visual.transformer.resblocks.8.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
849
+ "transformer.visual.transformer.resblocks.8.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
850
+ "transformer.visual.transformer.resblocks.8.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
851
+ "transformer.visual.transformer.resblocks.8.ln_1.bias": "pytorch_model-00002-of-00002.bin",
852
+ "transformer.visual.transformer.resblocks.8.ln_1.weight": "pytorch_model-00002-of-00002.bin",
853
+ "transformer.visual.transformer.resblocks.8.ln_2.bias": "pytorch_model-00002-of-00002.bin",
854
+ "transformer.visual.transformer.resblocks.8.ln_2.weight": "pytorch_model-00002-of-00002.bin",
855
+ "transformer.visual.transformer.resblocks.8.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
856
+ "transformer.visual.transformer.resblocks.8.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
857
+ "transformer.visual.transformer.resblocks.8.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
858
+ "transformer.visual.transformer.resblocks.8.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
859
+ "transformer.visual.transformer.resblocks.9.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
860
+ "transformer.visual.transformer.resblocks.9.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
861
+ "transformer.visual.transformer.resblocks.9.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
862
+ "transformer.visual.transformer.resblocks.9.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
863
+ "transformer.visual.transformer.resblocks.9.ln_1.bias": "pytorch_model-00002-of-00002.bin",
864
+ "transformer.visual.transformer.resblocks.9.ln_1.weight": "pytorch_model-00002-of-00002.bin",
865
+ "transformer.visual.transformer.resblocks.9.ln_2.bias": "pytorch_model-00002-of-00002.bin",
866
+ "transformer.visual.transformer.resblocks.9.ln_2.weight": "pytorch_model-00002-of-00002.bin",
867
+ "transformer.visual.transformer.resblocks.9.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
868
+ "transformer.visual.transformer.resblocks.9.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
869
+ "transformer.visual.transformer.resblocks.9.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
870
+ "transformer.visual.transformer.resblocks.9.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
871
+ "transformer.wte.weight": "pytorch_model-00001-of-00002.bin"
872
+ }
873
+ }
qwen.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "pad_token": "<|endoftext|>"
3
+ }
tokenization_qwen.py ADDED
@@ -0,0 +1,598 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Tokenization classes for QWen."""
7
+
8
+ import base64
9
+ import logging
10
+ import os
11
+ import requests
12
+ import unicodedata
13
+ from typing import Collection, Dict, List, Set, Tuple, Union, Any, Callable, Optional
14
+
15
+ import tiktoken
16
+ import numpy as np
17
+ from PIL import Image
18
+ from PIL import ImageFont
19
+ from PIL import ImageDraw
20
+ from transformers import PreTrainedTokenizer, AddedToken
21
+ from transformers.utils import try_to_load_from_cache
22
+
23
+ import matplotlib.colors as mcolors
24
+ from matplotlib.font_manager import FontProperties
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
30
+ FONT_PATH = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
31
+ if FONT_PATH is None:
32
+ if not os.path.exists("SimSun.ttf"):
33
+ ttf = requests.get("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/SimSun.ttf")
34
+ open("SimSun.ttf", "wb").write(ttf.content)
35
+ FONT_PATH = "SimSun.ttf"
36
+
37
+ PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
38
+ ENDOFTEXT = "<|endoftext|>"
39
+ IMSTART = "<|im_start|>"
40
+ IMEND = "<|im_end|>"
41
+ # as the default behavior is changed to allow special tokens in
42
+ # regular texts, the surface forms of special tokens need to be
43
+ # as different as possible to minimize the impact
44
+ EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
45
+ SPECIAL_TOKENS = (
46
+ ENDOFTEXT,
47
+ IMSTART,
48
+ IMEND,
49
+ ) + EXTRAS
50
+ IMG_TOKEN_SPAN = 256
51
+
52
+
53
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
54
+ with open(tiktoken_bpe_file, "rb") as f:
55
+ contents = f.read()
56
+ return {
57
+ base64.b64decode(token): int(rank)
58
+ for token, rank in (line.split() for line in contents.splitlines() if line)
59
+ }
60
+
61
+ def _list_find(
62
+ input_list: List[Any],
63
+ candidates: Tuple[Any],
64
+ start: int = 0,
65
+ ):
66
+ for i in range(start, len(input_list)):
67
+ if input_list[i] in candidates:
68
+ return i
69
+ return -1
70
+
71
+ def _replace_closed_tag(
72
+ input_tokens: List[Any],
73
+ start_tags: Union[Any, Tuple[Any]],
74
+ end_tags: Union[Any, Tuple[Any]],
75
+ inclusive_replace_func: Callable,
76
+ exclusive_replace_func: Callable = lambda x: x,
77
+ ):
78
+ if isinstance(start_tags, (str, int)):
79
+ start_tags = (start_tags,)
80
+ if isinstance(end_tags, (str, int)):
81
+ end_tags = (end_tags,)
82
+ assert len(start_tags) == len(end_tags)
83
+
84
+ output_tokens = []
85
+ end = 0
86
+ while True:
87
+ start = _list_find(input_tokens, start_tags, end)
88
+ if start == -1:
89
+ break
90
+ output_tokens.extend(exclusive_replace_func(input_tokens[end : start]))
91
+ tag_idx = start_tags.index(input_tokens[start])
92
+ end = _list_find(input_tokens, (end_tags[tag_idx],), start)
93
+ if end == -1:
94
+ raise ValueError("Unclosed image token")
95
+ output_tokens.extend(inclusive_replace_func(input_tokens[start : end + 1]))
96
+ end += 1
97
+ output_tokens.extend(exclusive_replace_func(input_tokens[end : ]))
98
+ return output_tokens
99
+
100
+ class QWenTokenizer(PreTrainedTokenizer):
101
+ """QWen tokenizer."""
102
+
103
+ vocab_files_names = VOCAB_FILES_NAMES
104
+
105
+ def __init__(
106
+ self,
107
+ vocab_file,
108
+ errors="replace",
109
+ image_start_tag='<img>',
110
+ image_end_tag='</img>',
111
+ image_pad_tag='<imgpad>',
112
+ ref_start_tag='<ref>',
113
+ ref_end_tag='</ref>',
114
+ box_start_tag='<box>',
115
+ box_end_tag='</box>',
116
+ quad_start_tag='<quad>',
117
+ quad_end_tag='</quad>',
118
+ **kwargs,
119
+ ):
120
+ super().__init__(**kwargs)
121
+ self.image_start_tag = image_start_tag
122
+ self.image_end_tag = image_end_tag
123
+ self.image_pad_tag = image_pad_tag
124
+ self.ref_start_tag = ref_start_tag
125
+ self.ref_end_tag = ref_end_tag
126
+ self.box_start_tag = box_start_tag
127
+ self.box_end_tag = box_end_tag
128
+ self.quad_start_tag = quad_start_tag
129
+ self.quad_end_tag = quad_end_tag
130
+ self.IMAGE_ST = (
131
+ ref_start_tag, ref_end_tag,
132
+ box_start_tag, box_end_tag,
133
+ quad_start_tag, quad_end_tag,
134
+ image_start_tag, image_end_tag,
135
+ image_pad_tag
136
+ )
137
+
138
+ self.errors = errors # how to handle errors in decoding
139
+
140
+ self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
141
+ self.special_tokens = {
142
+ token: index
143
+ for index, token in enumerate(
144
+ SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
145
+ )
146
+ }
147
+ self.img_start_id = self.special_tokens[self.image_start_tag]
148
+ self.img_end_id = self.special_tokens[self.image_end_tag]
149
+ self.img_pad_id = self.special_tokens[self.image_pad_tag]
150
+ self.ref_start_id = self.special_tokens[self.ref_start_tag]
151
+ self.ref_end_id = self.special_tokens[self.ref_end_tag]
152
+ self.box_start_id = self.special_tokens[self.box_start_tag]
153
+ self.box_end_id = self.special_tokens[self.box_end_tag]
154
+ self.quad_start_id = self.special_tokens[self.quad_start_tag]
155
+ self.quad_end_id = self.special_tokens[self.quad_end_tag]
156
+ self.image_special_tokens = set([
157
+ self.ref_start_id, self.ref_end_id, self.box_start_id, self.box_end_id,
158
+ self.quad_start_id, self.quad_end_id,
159
+ ])
160
+
161
+ enc = tiktoken.Encoding(
162
+ "Qwen",
163
+ pat_str=PAT_STR,
164
+ mergeable_ranks=self.mergeable_ranks,
165
+ special_tokens=self.special_tokens,
166
+ )
167
+ assert (
168
+ len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
169
+ ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
170
+
171
+ self.decoder = {
172
+ v: k for k, v in self.mergeable_ranks.items()
173
+ } # type: dict[int, bytes|str]
174
+ self.decoder.update({v: k for k, v in self.special_tokens.items()})
175
+
176
+ self.tokenizer = enc # type: tiktoken.Encoding
177
+
178
+ self.eod_id = self.tokenizer.eot_token
179
+ self.im_start_id = self.special_tokens[IMSTART]
180
+ self.im_end_id = self.special_tokens[IMEND]
181
+
182
+ def __getstate__(self):
183
+ # for pickle lovers
184
+ state = self.__dict__.copy()
185
+ del state['tokenizer']
186
+ return state
187
+
188
+ def __setstate__(self, state):
189
+ # tokenizer is not python native; don't pass it; rebuild it
190
+ self.__dict__.update(state)
191
+ enc = tiktoken.Encoding(
192
+ "Qwen",
193
+ pat_str=PAT_STR,
194
+ mergeable_ranks=self.mergeable_ranks,
195
+ special_tokens=self.special_tokens,
196
+ )
197
+ self.tokenizer = enc
198
+
199
+
200
+ def __len__(self) -> int:
201
+ return self.tokenizer.n_vocab
202
+
203
+ def get_vocab(self) -> Dict[bytes, int]:
204
+ return self.mergeable_ranks
205
+
206
+ def convert_tokens_to_ids(
207
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
208
+ ) -> List[int]:
209
+ ids = []
210
+ if isinstance(tokens, (str, bytes)):
211
+ if tokens in self.special_tokens:
212
+ return self.special_tokens[tokens]
213
+ else:
214
+ return self.mergeable_ranks.get(tokens)
215
+ for token in tokens:
216
+ if token in self.special_tokens:
217
+ ids.append(self.special_tokens[token])
218
+ else:
219
+ ids.append(self.mergeable_ranks.get(token))
220
+ return ids
221
+
222
+ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
223
+ if not special_tokens and new_tokens:
224
+ raise ValueError('Adding regular tokens is not supported')
225
+ for token in new_tokens:
226
+ surface_form = token.content if isinstance(token, AddedToken) else token
227
+ if surface_form not in SPECIAL_TOKENS + self.IMAGE_ST:
228
+ raise ValueError('Adding unknown special tokens is not supported')
229
+ return 0
230
+
231
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
232
+ """
233
+ Save only the vocabulary of the tokenizer (vocabulary).
234
+
235
+ Returns:
236
+ `Tuple(str)`: Paths to the files saved.
237
+ """
238
+ file_path = os.path.join(save_directory, "qwen.tiktoken")
239
+ with open(file_path, "w", encoding="utf8") as w:
240
+ for k, v in self.mergeable_ranks.items():
241
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
242
+ w.write(line)
243
+ return (file_path,)
244
+
245
+ def tokenize(
246
+ self,
247
+ text: str,
248
+ allowed_special: Union[Set, str] = "all",
249
+ disallowed_special: Union[Collection, str] = (),
250
+ **kwargs,
251
+ ) -> List[Union[bytes, str]]:
252
+ """
253
+ Converts a string in a sequence of tokens.
254
+
255
+ Args:
256
+ text (`str`):
257
+ The sequence to be encoded.
258
+ allowed_special (`Literal["all"]` or `set`):
259
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
260
+ Default to "all".
261
+ disallowed_special (`Literal["all"]` or `Collection`):
262
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
263
+ Default to an empty tuple.
264
+
265
+ kwargs (additional keyword arguments, *optional*):
266
+ Will be passed to the underlying model specific encode method.
267
+
268
+ Returns:
269
+ `List[bytes|str]`: The list of tokens.
270
+ """
271
+ tokens = []
272
+ text = unicodedata.normalize("NFC", text)
273
+
274
+ # this implementation takes a detour: text -> token id -> token surface forms
275
+ for t in self.tokenizer.encode(
276
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
277
+ ):
278
+ tokens.append(self.decoder[t])
279
+
280
+ def _encode_imgurl(img_tokens):
281
+ assert img_tokens[0] == self.image_start_tag and img_tokens[-1] == self.image_end_tag
282
+ img_tokens = img_tokens[1:-1]
283
+ img_url = b''.join(img_tokens)
284
+ out_img_tokens = list(map(self.decoder.get, img_url))
285
+ if len(out_img_tokens) > IMG_TOKEN_SPAN:
286
+ raise ValueError("The content in {}..{} is too long".format(
287
+ self.image_start_tag, self.image_end_tag))
288
+ out_img_tokens.extend([self.image_pad_tag] * (IMG_TOKEN_SPAN - len(out_img_tokens)))
289
+ out_img_tokens = [self.image_start_tag] + out_img_tokens + [self.image_end_tag]
290
+ return out_img_tokens
291
+
292
+ return _replace_closed_tag(tokens, self.image_start_tag, self.image_end_tag, _encode_imgurl)
293
+
294
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
295
+ """
296
+ Converts a sequence of tokens in a single string.
297
+ """
298
+ text = ""
299
+ temp = b""
300
+ for t in tokens:
301
+ if isinstance(t, str):
302
+ if temp:
303
+ text += temp.decode("utf-8", errors=self.errors)
304
+ temp = b""
305
+ text += t
306
+ elif isinstance(t, bytes):
307
+ temp += t
308
+ else:
309
+ raise TypeError("token should only be of type types or str")
310
+ if temp:
311
+ text += temp.decode("utf-8", errors=self.errors)
312
+ return text
313
+
314
+ @property
315
+ def vocab_size(self):
316
+ return self.tokenizer.n_vocab
317
+
318
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
319
+ """Converts an id to a token, special tokens included"""
320
+ if index in self.decoder:
321
+ return self.decoder[index]
322
+ raise ValueError("unknown ids")
323
+
324
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
325
+ """Converts a token to an id using the vocab, special tokens included"""
326
+ if token in self.special_tokens:
327
+ return self.special_tokens[token]
328
+ if token in self.mergeable_ranks:
329
+ return self.mergeable_ranks[token]
330
+ raise ValueError("unknown token")
331
+
332
+ def _tokenize(self, text: str, **kwargs):
333
+ """
334
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
335
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
336
+
337
+ Do NOT take care of added tokens.
338
+ """
339
+ raise NotImplementedError
340
+
341
+ def _decode(
342
+ self,
343
+ token_ids: Union[int, List[int]],
344
+ skip_special_tokens: bool = False,
345
+ errors: str = None,
346
+ **kwargs,
347
+ ) -> str:
348
+ if isinstance(token_ids, int):
349
+ token_ids = [token_ids]
350
+
351
+ def _decode_imgurl(img_token_ids):
352
+ assert img_token_ids[0] == self.img_start_id and img_token_ids[-1] == self.img_end_id
353
+ img_token_ids = img_token_ids[1:-1]
354
+ img_token_ids = img_token_ids[ : img_token_ids.index(self.img_pad_id)]
355
+ img_url = bytes(img_token_ids).decode('utf-8')
356
+ return [self.img_start_id] + self.tokenizer.encode(img_url) + [self.img_end_id]
357
+
358
+ token_ids = _replace_closed_tag(token_ids, self.img_start_id, self.img_end_id, _decode_imgurl)
359
+
360
+ if skip_special_tokens:
361
+ if kwargs.get('keep_image_special', False):
362
+ token_ids = [i for i in token_ids if i < self.eod_id
363
+ or i in self.image_special_tokens]
364
+ else:
365
+ token_ids = [i for i in token_ids if i < self.eod_id]
366
+ return self.tokenizer.decode(token_ids, errors=errors or self.errors)
367
+
368
+ def to_list_format(self, text: str):
369
+ text = unicodedata.normalize("NFC", text)
370
+ token_ids = self.tokenizer.encode(
371
+ text, allowed_special=set(self.IMAGE_ST + (ENDOFTEXT,)))
372
+
373
+ def _encode_vl_info(tokens):
374
+ if len(tokens) == 0:
375
+ return []
376
+ if tokens[0] == self.img_start_id and tokens[-1] == self.img_end_id:
377
+ key = 'image'
378
+ elif tokens[0] == self.ref_start_id and tokens[-1] == self.ref_end_id:
379
+ key = 'ref'
380
+ elif tokens[0] == self.box_start_id and tokens[-1] == self.box_end_id:
381
+ key = 'box'
382
+ elif tokens[0] == self.quad_start_id and tokens[-1] == self.quad_end_id:
383
+ key = 'quad'
384
+ else:
385
+ _tobytes = lambda x: x.encode('utf-8') if isinstance(x, str) else x
386
+ return [{'text': b''.join(map(_tobytes, map(self.decoder.get, tokens))).decode('utf-8')}]
387
+ _tobytes = lambda x: x.encode('utf-8') if isinstance(x, str) else x
388
+ val = b''.join(map(_tobytes, map(self.decoder.get, tokens[1:-1]))).decode('utf-8')
389
+ return [{key: val}]
390
+
391
+ return _replace_closed_tag(
392
+ token_ids,
393
+ (self.img_start_id, self.ref_start_id, self.box_start_id, self.quad_start_id),
394
+ (self.img_end_id, self.ref_end_id, self.box_end_id, self.quad_end_id),
395
+ _encode_vl_info,
396
+ _encode_vl_info,
397
+ )
398
+
399
+ def from_list_format(self, list_format: List[Dict]):
400
+ text = ''
401
+ num_images = 0
402
+ for ele in list_format:
403
+ if 'image' in ele:
404
+ num_images += 1
405
+ text += f'Picture {num_images}: '
406
+ text += self.image_start_tag + ele['image'] + self.image_end_tag
407
+ text += '\n'
408
+ elif 'text' in ele:
409
+ text += ele['text']
410
+ elif 'box' in ele:
411
+ if 'ref' in ele:
412
+ text += self.ref_start_tag + ele['ref'] + self.ref_end_tag
413
+ for box in ele['box']:
414
+ text += self.box_start_tag + '(%d,%d),(%d,%d)' % (box[0], box[1], box[2], box[3]) + self.box_end_tag
415
+ else:
416
+ raise ValueError("Unsupport element: " + str(ele))
417
+ return text
418
+
419
+ def _fetch_latest_picture(self, response, history):
420
+ if history is None:
421
+ history = []
422
+ _history = history + [(response, None)]
423
+ for q, r in _history[::-1]:
424
+ for ele in self.to_list_format(q)[::-1]:
425
+ if 'image' in ele:
426
+ return ele['image']
427
+ return None
428
+
429
+ def _fetch_all_box_with_ref(self, text):
430
+ list_format = self.to_list_format(text)
431
+ output = []
432
+ for i, ele in enumerate(list_format):
433
+ if 'box' in ele:
434
+ bbox = tuple(map(int, ele['box'].replace('(', '').replace(')', '').split(',')))
435
+ assert len(bbox) == 4
436
+ output.append({'box': bbox})
437
+ if i > 0 and 'ref' in list_format[i-1]:
438
+ output[-1]['ref'] = list_format[i-1]['ref'].strip()
439
+ return output
440
+
441
+ def draw_bbox_on_latest_picture(
442
+ self,
443
+ response,
444
+ history=None,
445
+ ) -> Optional[Image.Image]:
446
+ image = self._fetch_latest_picture(response, history)
447
+ if image is None:
448
+ return None
449
+ if image.startswith("http://") or image.startswith("https://"):
450
+ image = Image.open(requests.get(image, stream=True).raw).convert("RGB")
451
+ h, w = image.height, image.width
452
+ else:
453
+ image = np.asarray(Image.open(image).convert("RGB"))
454
+ h, w = image.shape[0], image.shape[1]
455
+ visualizer = Visualizer(image)
456
+
457
+ boxes = self._fetch_all_box_with_ref(response)
458
+ if not boxes:
459
+ return None
460
+ color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()]) # init color
461
+ for box in boxes:
462
+ if 'ref' in box: # random new color for new refexps
463
+ color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()])
464
+ x1, y1, x2, y2 = box['box']
465
+ x1, y1, x2, y2 = (int(x1 / 1000 * w), int(y1 / 1000 * h), int(x2 / 1000 * w), int(y2 / 1000 * h))
466
+ visualizer.draw_box((x1, y1, x2, y2), alpha=1, edge_color=color)
467
+ if 'ref' in box:
468
+ visualizer.draw_text(box['ref'], (x1, y1), color=color, horizontal_alignment="left")
469
+ return visualizer.output
470
+
471
+
472
+ import colorsys
473
+ import logging
474
+ import math
475
+ import numpy as np
476
+ import matplotlib as mpl
477
+ import matplotlib.colors as mplc
478
+ import matplotlib.figure as mplfigure
479
+ import torch
480
+ from matplotlib.backends.backend_agg import FigureCanvasAgg
481
+ from PIL import Image
482
+ import random
483
+
484
+ logger = logging.getLogger(__name__)
485
+
486
+
487
+ class VisImage:
488
+ def __init__(self, img, scale=1.0):
489
+ self.img = img
490
+ self.scale = scale
491
+ self.width, self.height = img.shape[1], img.shape[0]
492
+ self._setup_figure(img)
493
+
494
+ def _setup_figure(self, img):
495
+ fig = mplfigure.Figure(frameon=False)
496
+ self.dpi = fig.get_dpi()
497
+ # add a small 1e-2 to avoid precision lost due to matplotlib's truncation
498
+ # (https://github.com/matplotlib/matplotlib/issues/15363)
499
+ fig.set_size_inches(
500
+ (self.width * self.scale + 1e-2) / self.dpi,
501
+ (self.height * self.scale + 1e-2) / self.dpi,
502
+ )
503
+ self.canvas = FigureCanvasAgg(fig)
504
+ # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig)
505
+ ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
506
+ ax.axis("off")
507
+ self.fig = fig
508
+ self.ax = ax
509
+ self.reset_image(img)
510
+
511
+ def reset_image(self, img):
512
+ img = img.astype("uint8")
513
+ self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest")
514
+
515
+ def save(self, filepath):
516
+ self.fig.savefig(filepath)
517
+
518
+ def get_image(self):
519
+ canvas = self.canvas
520
+ s, (width, height) = canvas.print_to_buffer()
521
+
522
+ buffer = np.frombuffer(s, dtype="uint8")
523
+
524
+ img_rgba = buffer.reshape(height, width, 4)
525
+ rgb, alpha = np.split(img_rgba, [3], axis=2)
526
+ return rgb.astype("uint8")
527
+
528
+
529
+ class Visualizer:
530
+ def __init__(self, img_rgb, metadata=None, scale=1.0):
531
+ self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)
532
+ self.font_path = FONT_PATH
533
+ self.output = VisImage(self.img, scale=scale)
534
+ self.cpu_device = torch.device("cpu")
535
+
536
+ # too small texts are useless, therefore clamp to 14
537
+ self._default_font_size = max(
538
+ np.sqrt(self.output.height * self.output.width) // 30, 15 // scale
539
+ )
540
+
541
+ def draw_text(
542
+ self,
543
+ text,
544
+ position,
545
+ *,
546
+ font_size=None,
547
+ color="g",
548
+ horizontal_alignment="center",
549
+ rotation=0,
550
+ ):
551
+ if not font_size:
552
+ font_size = self._default_font_size
553
+
554
+ # since the text background is dark, we don't want the text to be dark
555
+ color = np.maximum(list(mplc.to_rgb(color)), 0.2)
556
+ color[np.argmax(color)] = max(0.8, np.max(color))
557
+
558
+ x, y = position
559
+ self.output.ax.text(
560
+ x,
561
+ y,
562
+ text,
563
+ size=font_size * self.output.scale,
564
+ fontproperties=FontProperties(fname=self.font_path),
565
+ bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"},
566
+ verticalalignment="top",
567
+ horizontalalignment=horizontal_alignment,
568
+ color=color,
569
+ zorder=10,
570
+ rotation=rotation,
571
+ )
572
+ return self.output
573
+
574
+ def draw_box(self, box_coord, alpha=0.5, edge_color="g", line_style="-"):
575
+
576
+ x0, y0, x1, y1 = box_coord
577
+ width = x1 - x0
578
+ height = y1 - y0
579
+
580
+ linewidth = max(self._default_font_size / 4, 1)
581
+
582
+ self.output.ax.add_patch(
583
+ mpl.patches.Rectangle(
584
+ (x0, y0),
585
+ width,
586
+ height,
587
+ fill=False,
588
+ edgecolor=edge_color,
589
+ linewidth=linewidth * self.output.scale,
590
+ alpha=alpha,
591
+ linestyle=line_style,
592
+ )
593
+ )
594
+ return self.output
595
+
596
+ def get_output(self):
597
+
598
+ return self.output
tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "tokenization_qwen.QWenTokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "clean_up_tokenization_spaces": true,
9
+ "model_max_length": 800,
10
+ "padding_side": "right",
11
+ "tokenizer_class": "QWenTokenizer"
12
+ }
zero_to_fp32.py ADDED
@@ -0,0 +1,587 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example: python zero_to_fp32.py . pytorch_model.bin
14
+
15
+ import argparse
16
+ import torch
17
+ import glob
18
+ import math
19
+ import os
20
+ import re
21
+ from collections import OrderedDict
22
+ from dataclasses import dataclass
23
+
24
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
25
+ # DeepSpeed data structures it has to be available in the current python environment.
26
+ from deepspeed.utils import logger
27
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
28
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
29
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
30
+
31
+
32
+ @dataclass
33
+ class zero_model_state:
34
+ buffers: dict()
35
+ param_shapes: dict()
36
+ shared_params: list
37
+ ds_version: int
38
+ frozen_param_shapes: dict()
39
+ frozen_param_fragments: dict()
40
+
41
+
42
+ debug = 0
43
+
44
+ # load to cpu
45
+ device = torch.device('cpu')
46
+
47
+
48
+ def atoi(text):
49
+ return int(text) if text.isdigit() else text
50
+
51
+
52
+ def natural_keys(text):
53
+ '''
54
+ alist.sort(key=natural_keys) sorts in human order
55
+ http://nedbatchelder.com/blog/200712/human_sorting.html
56
+ (See Toothy's implementation in the comments)
57
+ '''
58
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
59
+
60
+
61
+ def get_model_state_file(checkpoint_dir, zero_stage):
62
+ if not os.path.isdir(checkpoint_dir):
63
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
64
+
65
+ # there should be only one file
66
+ if zero_stage <= 2:
67
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
68
+ elif zero_stage == 3:
69
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
70
+
71
+ if not os.path.exists(file):
72
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
73
+
74
+ return file
75
+
76
+
77
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
78
+ # XXX: need to test that this simple glob rule works for multi-node setup too
79
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
80
+
81
+ if len(ckpt_files) == 0:
82
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
83
+
84
+ return ckpt_files
85
+
86
+
87
+ def get_optim_files(checkpoint_dir):
88
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
89
+
90
+
91
+ def get_model_state_files(checkpoint_dir):
92
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
93
+
94
+
95
+ def parse_model_states(files):
96
+ zero_model_states = []
97
+ for file in files:
98
+ state_dict = torch.load(file, map_location=device)
99
+
100
+ if BUFFER_NAMES not in state_dict:
101
+ raise ValueError(f"{file} is not a model state checkpoint")
102
+ buffer_names = state_dict[BUFFER_NAMES]
103
+ if debug:
104
+ print("Found buffers:", buffer_names)
105
+
106
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
107
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
108
+ param_shapes = state_dict[PARAM_SHAPES]
109
+
110
+ # collect parameters that are included in param_shapes
111
+ param_names = []
112
+ for s in param_shapes:
113
+ for name in s.keys():
114
+ param_names.append(name)
115
+
116
+ # update with frozen parameters
117
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
118
+ if frozen_param_shapes is not None:
119
+ if debug:
120
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
121
+ param_names += list(frozen_param_shapes.keys())
122
+
123
+ # handle shared params
124
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
125
+
126
+ ds_version = state_dict.get(DS_VERSION, None)
127
+
128
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
129
+
130
+ z_model_state = zero_model_state(buffers=buffers,
131
+ param_shapes=param_shapes,
132
+ shared_params=shared_params,
133
+ ds_version=ds_version,
134
+ frozen_param_shapes=frozen_param_shapes,
135
+ frozen_param_fragments=frozen_param_fragments)
136
+ zero_model_states.append(z_model_state)
137
+
138
+ return zero_model_states
139
+
140
+
141
+ def parse_optim_states(files, ds_checkpoint_dir):
142
+
143
+ total_files = len(files)
144
+ state_dicts = []
145
+ for f in files:
146
+ state_dict = torch.load(f, map_location=device)
147
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
148
+ # and also handle the case where it was already removed by another helper script
149
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
150
+ state_dicts.append(state_dict)
151
+
152
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
153
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
154
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
155
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
156
+
157
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
158
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
159
+ # use the max of the partition_count to get the dp world_size.
160
+
161
+ if type(world_size) is list:
162
+ world_size = max(world_size)
163
+
164
+ if world_size != total_files:
165
+ raise ValueError(
166
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
167
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
168
+ )
169
+
170
+ # the groups are named differently in each stage
171
+ if zero_stage <= 2:
172
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
173
+ elif zero_stage == 3:
174
+ fp32_groups_key = FP32_FLAT_GROUPS
175
+ else:
176
+ raise ValueError(f"unknown zero stage {zero_stage}")
177
+
178
+ if zero_stage <= 2:
179
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
180
+ elif zero_stage == 3:
181
+ # if there is more than one param group, there will be multiple flattened tensors - one
182
+ # flattened tensor per group - for simplicity merge them into a single tensor
183
+ #
184
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
185
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
186
+
187
+ fp32_flat_groups = [
188
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
189
+ ]
190
+
191
+ return zero_stage, world_size, fp32_flat_groups
192
+
193
+
194
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
195
+ """
196
+ Returns fp32 state_dict reconstructed from ds checkpoint
197
+
198
+ Args:
199
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
200
+
201
+ """
202
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
203
+
204
+ optim_files = get_optim_files(ds_checkpoint_dir)
205
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
206
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
207
+
208
+ model_files = get_model_state_files(ds_checkpoint_dir)
209
+
210
+ zero_model_states = parse_model_states(model_files)
211
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
212
+
213
+ if zero_stage <= 2:
214
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
215
+ elif zero_stage == 3:
216
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
217
+
218
+
219
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
220
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
221
+ return
222
+
223
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
224
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
225
+
226
+ if debug:
227
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
228
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
229
+
230
+ wanted_params = len(frozen_param_shapes)
231
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
232
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
233
+ print(f'Frozen params: Have {avail_numel} numels to process.')
234
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
235
+
236
+ total_params = 0
237
+ total_numel = 0
238
+ for name, shape in frozen_param_shapes.items():
239
+ total_params += 1
240
+ unpartitioned_numel = shape.numel()
241
+ total_numel += unpartitioned_numel
242
+
243
+ state_dict[name] = frozen_param_fragments[name]
244
+
245
+ if debug:
246
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
247
+
248
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
249
+
250
+
251
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
252
+ param_shapes = zero_model_states[0].param_shapes
253
+
254
+ # Reconstruction protocol:
255
+ #
256
+ # XXX: document this
257
+
258
+ if debug:
259
+ for i in range(world_size):
260
+ for j in range(len(fp32_flat_groups[0])):
261
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
262
+
263
+ # XXX: memory usage doubles here (zero2)
264
+ num_param_groups = len(fp32_flat_groups[0])
265
+ merged_single_partition_of_fp32_groups = []
266
+ for i in range(num_param_groups):
267
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
268
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
269
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
270
+ avail_numel = sum(
271
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
272
+
273
+ if debug:
274
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
275
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
276
+ # not asserting if there is a mismatch due to possible padding
277
+ print(f"Have {avail_numel} numels to process.")
278
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
279
+
280
+ # params
281
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
282
+ # out-of-core computing solution
283
+ total_numel = 0
284
+ total_params = 0
285
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
286
+ offset = 0
287
+ avail_numel = full_single_fp32_vector.numel()
288
+ for name, shape in shapes.items():
289
+
290
+ unpartitioned_numel = shape.numel()
291
+ total_numel += unpartitioned_numel
292
+ total_params += 1
293
+
294
+ if debug:
295
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
296
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
297
+ offset += unpartitioned_numel
298
+
299
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
300
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
301
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
302
+ # live optimizer object, so we are checking that the numbers are within the right range
303
+ align_to = 2 * world_size
304
+
305
+ def zero2_align(x):
306
+ return align_to * math.ceil(x / align_to)
307
+
308
+ if debug:
309
+ print(f"original offset={offset}, avail_numel={avail_numel}")
310
+
311
+ offset = zero2_align(offset)
312
+ avail_numel = zero2_align(avail_numel)
313
+
314
+ if debug:
315
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
316
+
317
+ # Sanity check
318
+ if offset != avail_numel:
319
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
320
+
321
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
322
+
323
+
324
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
325
+ state_dict = OrderedDict()
326
+
327
+ # buffers
328
+ buffers = zero_model_states[0].buffers
329
+ state_dict.update(buffers)
330
+ if debug:
331
+ print(f"added {len(buffers)} buffers")
332
+
333
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
334
+
335
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
336
+
337
+ # recover shared parameters
338
+ for pair in zero_model_states[0].shared_params:
339
+ if pair[1] in state_dict:
340
+ state_dict[pair[0]] = state_dict[pair[1]]
341
+
342
+ return state_dict
343
+
344
+
345
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
346
+ remainder = unpartitioned_numel % world_size
347
+ padding_numel = (world_size - remainder) if remainder else 0
348
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
349
+ return partitioned_numel, padding_numel
350
+
351
+
352
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
353
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
354
+ return
355
+
356
+ if debug:
357
+ for i in range(world_size):
358
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
359
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
360
+
361
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
362
+ wanted_params = len(frozen_param_shapes)
363
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
364
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
365
+ print(f'Frozen params: Have {avail_numel} numels to process.')
366
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
367
+
368
+ total_params = 0
369
+ total_numel = 0
370
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
371
+ total_params += 1
372
+ unpartitioned_numel = shape.numel()
373
+ total_numel += unpartitioned_numel
374
+
375
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
376
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
377
+
378
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
379
+
380
+ if debug:
381
+ print(
382
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
383
+ )
384
+
385
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
386
+
387
+
388
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
389
+ param_shapes = zero_model_states[0].param_shapes
390
+ avail_numel = fp32_flat_groups[0].numel() * world_size
391
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
392
+ # param, re-consolidating each param, while dealing with padding if any
393
+
394
+ # merge list of dicts, preserving order
395
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
396
+
397
+ if debug:
398
+ for i in range(world_size):
399
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
400
+
401
+ wanted_params = len(param_shapes)
402
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
403
+ # not asserting if there is a mismatch due to possible padding
404
+ avail_numel = fp32_flat_groups[0].numel() * world_size
405
+ print(f"Trainable params: Have {avail_numel} numels to process.")
406
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
407
+
408
+ # params
409
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
410
+ # out-of-core computing solution
411
+ offset = 0
412
+ total_numel = 0
413
+ total_params = 0
414
+ for name, shape in param_shapes.items():
415
+
416
+ unpartitioned_numel = shape.numel()
417
+ total_numel += unpartitioned_numel
418
+ total_params += 1
419
+
420
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
421
+
422
+ if debug:
423
+ print(
424
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
425
+ )
426
+
427
+ # XXX: memory usage doubles here
428
+ state_dict[name] = torch.cat(
429
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
430
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
431
+ offset += partitioned_numel
432
+
433
+ offset *= world_size
434
+
435
+ # Sanity check
436
+ if offset != avail_numel:
437
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
438
+
439
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
440
+
441
+
442
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
443
+ state_dict = OrderedDict()
444
+
445
+ # buffers
446
+ buffers = zero_model_states[0].buffers
447
+ state_dict.update(buffers)
448
+ if debug:
449
+ print(f"added {len(buffers)} buffers")
450
+
451
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
452
+
453
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
454
+
455
+ # recover shared parameters
456
+ for pair in zero_model_states[0].shared_params:
457
+ if pair[1] in state_dict:
458
+ state_dict[pair[0]] = state_dict[pair[1]]
459
+
460
+ return state_dict
461
+
462
+
463
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
464
+ """
465
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
466
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
467
+ via a model hub.
468
+
469
+ Args:
470
+ - ``checkpoint_dir``: path to the desired checkpoint folder
471
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
472
+
473
+ Returns:
474
+ - pytorch ``state_dict``
475
+
476
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
477
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
478
+ the checkpoint.
479
+
480
+ A typical usage might be ::
481
+
482
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
483
+ # do the training and checkpoint saving
484
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
485
+ model = model.cpu() # move to cpu
486
+ model.load_state_dict(state_dict)
487
+ # submit to model hub or save the model to share with others
488
+
489
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
490
+ application. i.e. you will need to re-initialize the deepspeed engine, since
491
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
492
+
493
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
494
+
495
+ """
496
+ if tag is None:
497
+ latest_path = os.path.join(checkpoint_dir, 'latest')
498
+ if os.path.isfile(latest_path):
499
+ with open(latest_path, 'r') as fd:
500
+ tag = fd.read().strip()
501
+ else:
502
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
503
+
504
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
505
+
506
+ if not os.path.isdir(ds_checkpoint_dir):
507
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
508
+
509
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
510
+
511
+
512
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
513
+ """
514
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
515
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
516
+
517
+ Args:
518
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
519
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
520
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
521
+ """
522
+
523
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
524
+ print(f"Saving fp32 state dict to {output_file}")
525
+ torch.save(state_dict, output_file)
526
+
527
+
528
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
529
+ """
530
+ 1. Put the provided model to cpu
531
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
532
+ 3. Load it into the provided model
533
+
534
+ Args:
535
+ - ``model``: the model object to update
536
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
537
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
538
+
539
+ Returns:
540
+ - ``model`: modified model
541
+
542
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
543
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
544
+ conveniently placed for you in the checkpoint folder.
545
+
546
+ A typical usage might be ::
547
+
548
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
549
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
550
+ # submit to model hub or save the model to share with others
551
+
552
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
553
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
554
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
555
+
556
+ """
557
+ logger.info(f"Extracting fp32 weights")
558
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
559
+
560
+ logger.info(f"Overwriting model with fp32 weights")
561
+ model = model.cpu()
562
+ model.load_state_dict(state_dict, strict=False)
563
+
564
+ return model
565
+
566
+
567
+ if __name__ == "__main__":
568
+
569
+ parser = argparse.ArgumentParser()
570
+ parser.add_argument("checkpoint_dir",
571
+ type=str,
572
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
573
+ parser.add_argument(
574
+ "output_file",
575
+ type=str,
576
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
577
+ parser.add_argument("-t",
578
+ "--tag",
579
+ type=str,
580
+ default=None,
581
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
582
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
583
+ args = parser.parse_args()
584
+
585
+ debug = args.debug
586
+
587
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)