teowu commited on
Commit
de64719
1 Parent(s): 9cea3e4

Upload configuration_mplug_owl2.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. configuration_mplug_owl2.py +334 -0
configuration_mplug_owl2.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+ import copy
6
+ import os
7
+ from typing import Union
8
+
9
+ from transformers.configuration_utils import PretrainedConfig
10
+ from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
11
+ from transformers.utils import logging
12
+ from transformers.models.auto import CONFIG_MAPPING
13
+
14
+
15
+ class LlamaConfig(PretrainedConfig):
16
+ r"""
17
+ This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA
18
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
19
+ defaults will yield a similar configuration to that of the LLaMA-7B.
20
+
21
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
22
+ documentation from [`PretrainedConfig`] for more information.
23
+
24
+
25
+ Args:
26
+ vocab_size (`int`, *optional*, defaults to 32000):
27
+ Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
28
+ `inputs_ids` passed when calling [`LlamaModel`]
29
+ hidden_size (`int`, *optional*, defaults to 4096):
30
+ Dimension of the hidden representations.
31
+ intermediate_size (`int`, *optional*, defaults to 11008):
32
+ Dimension of the MLP representations.
33
+ num_hidden_layers (`int`, *optional*, defaults to 32):
34
+ Number of hidden layers in the Transformer decoder.
35
+ num_attention_heads (`int`, *optional*, defaults to 32):
36
+ Number of attention heads for each attention layer in the Transformer decoder.
37
+ num_key_value_heads (`int`, *optional*):
38
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
39
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
40
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
41
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
42
+ by meanpooling all the original heads within that group. For more details checkout [this
43
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
44
+ `num_attention_heads`.
45
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
46
+ The non-linear activation function (function or string) in the decoder.
47
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
48
+ The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
49
+ Llama 2 up to 4096, CodeLlama up to 16384.
50
+ initializer_range (`float`, *optional*, defaults to 0.02):
51
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
52
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
53
+ The epsilon used by the rms normalization layers.
54
+ use_cache (`bool`, *optional*, defaults to `True`):
55
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
56
+ relevant if `config.is_decoder=True`.
57
+ pad_token_id (`int`, *optional*):
58
+ Padding token id.
59
+ bos_token_id (`int`, *optional*, defaults to 1):
60
+ Beginning of stream token id.
61
+ eos_token_id (`int`, *optional*, defaults to 2):
62
+ End of stream token id.
63
+ pretraining_tp (`int`, *optional*, defaults to 1):
64
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
65
+ document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
66
+ necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
67
+ issue](https://github.com/pytorch/pytorch/issues/76232).
68
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
69
+ Whether to tie weight embeddings
70
+ rope_theta (`float`, *optional*, defaults to 10000.0):
71
+ The base period of the RoPE embeddings.
72
+ rope_scaling (`Dict`, *optional*):
73
+ Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
74
+ strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
75
+ `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
76
+ `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
77
+ these scaling strategies behave:
78
+ https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
79
+ experimental feature, subject to breaking API changes in future versions.
80
+ attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
81
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
82
+
83
+
84
+ ```python
85
+ >>> from transformers import LlamaModel, LlamaConfig
86
+
87
+ >>> # Initializing a LLaMA llama-7b style configuration
88
+ >>> configuration = LlamaConfig()
89
+
90
+ >>> # Initializing a model from the llama-7b style configuration
91
+ >>> model = LlamaModel(configuration)
92
+
93
+ >>> # Accessing the model configuration
94
+ >>> configuration = model.config
95
+ ```"""
96
+ model_type = "llama"
97
+ keys_to_ignore_at_inference = ["past_key_values"]
98
+
99
+ def __init__(
100
+ self,
101
+ vocab_size=32000,
102
+ hidden_size=4096,
103
+ intermediate_size=11008,
104
+ num_hidden_layers=32,
105
+ num_attention_heads=32,
106
+ num_key_value_heads=None,
107
+ hidden_act="silu",
108
+ max_position_embeddings=2048,
109
+ initializer_range=0.02,
110
+ rms_norm_eps=1e-6,
111
+ use_cache=True,
112
+ pad_token_id=None,
113
+ bos_token_id=1,
114
+ eos_token_id=2,
115
+ pretraining_tp=1,
116
+ tie_word_embeddings=False,
117
+ rope_theta=10000.0,
118
+ rope_scaling=None,
119
+ attention_bias=False,
120
+ attention_dropout=0.0,
121
+ **kwargs,
122
+ ):
123
+ self.vocab_size = vocab_size
124
+ self.max_position_embeddings = max_position_embeddings
125
+ self.hidden_size = hidden_size
126
+ self.intermediate_size = intermediate_size
127
+ self.num_hidden_layers = num_hidden_layers
128
+ self.num_attention_heads = num_attention_heads
129
+
130
+ # for backward compatibility
131
+ if num_key_value_heads is None:
132
+ num_key_value_heads = num_attention_heads
133
+
134
+ self.num_key_value_heads = num_key_value_heads
135
+ self.hidden_act = hidden_act
136
+ self.initializer_range = initializer_range
137
+ self.rms_norm_eps = rms_norm_eps
138
+ self.pretraining_tp = pretraining_tp
139
+ self.use_cache = use_cache
140
+ self.rope_theta = rope_theta
141
+ self.rope_scaling = rope_scaling
142
+ self._rope_scaling_validation()
143
+ self.attention_bias = attention_bias
144
+ self.attention_dropout = attention_dropout
145
+
146
+ super().__init__(
147
+ pad_token_id=pad_token_id,
148
+ bos_token_id=bos_token_id,
149
+ eos_token_id=eos_token_id,
150
+ tie_word_embeddings=tie_word_embeddings,
151
+ **kwargs,
152
+ )
153
+
154
+ def _rope_scaling_validation(self):
155
+ """
156
+ Validate the `rope_scaling` configuration.
157
+ """
158
+ if self.rope_scaling is None:
159
+ return
160
+
161
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
162
+ raise ValueError(
163
+ "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
164
+ f"got {self.rope_scaling}"
165
+ )
166
+ rope_scaling_type = self.rope_scaling.get("type", None)
167
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
168
+ if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
169
+ raise ValueError(
170
+ f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
171
+ )
172
+ if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
173
+ raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")
174
+
175
+
176
+ class MplugOwlVisionConfig(PretrainedConfig):
177
+ r"""
178
+ This is the configuration class to store the configuration of a [`MplugOwlVisionModel`]. It is used to instantiate
179
+ a
180
+ mPLUG-Owl vision encoder according to the specified arguments, defining the model architecture. Instantiating a
181
+ configuration defaults will yield a similar configuration to that of the mPLUG-Owl
182
+ [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
183
+
184
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
185
+ documentation from [`PretrainedConfig`] for more information.
186
+
187
+ Args:
188
+ hidden_size (`int`, *optional*, defaults to 768):
189
+ Dimensionality of the encoder layers and the pooler layer.
190
+ intermediate_size (`int`, *optional*, defaults to 3072):
191
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
192
+ num_hidden_layers (`int`, *optional*, defaults to 12):
193
+ Number of hidden layers in the Transformer encoder.
194
+ num_attention_heads (`int`, *optional*, defaults to 12):
195
+ Number of attention heads for each attention layer in the Transformer encoder.
196
+ image_size (`int`, *optional*, defaults to 224):
197
+ The size (resolution) of each image.
198
+ patch_size (`int`, *optional*, defaults to 32):
199
+ The size (resolution) of each patch.
200
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
201
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
202
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
203
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
204
+ The epsilon used by the layer normalization layers.
205
+ attention_dropout (`float`, *optional*, defaults to 0.0):
206
+ The dropout ratio for the attention probabilities.
207
+ initializer_range (`float`, *optional*, defaults to 0.02):
208
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
209
+ initializer_factor (`float`, *optional*, defaults to 1):
210
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
211
+ testing).
212
+
213
+
214
+ ```"""
215
+
216
+ model_type = "mplug_owl_vision_model"
217
+
218
+ def __init__(
219
+ self,
220
+ hidden_size=1024,
221
+ intermediate_size=4096,
222
+ projection_dim=768,
223
+ num_hidden_layers=24,
224
+ num_attention_heads=16,
225
+ num_channels=3,
226
+ image_size=448,
227
+ patch_size=14,
228
+ hidden_act="quick_gelu",
229
+ layer_norm_eps=1e-6,
230
+ attention_dropout=0.0,
231
+ initializer_range=0.02,
232
+ initializer_factor=1.0,
233
+ use_flash_attn=False,
234
+ **kwargs,
235
+ ):
236
+ super().__init__(**kwargs)
237
+ self.hidden_size = hidden_size
238
+ self.intermediate_size = intermediate_size
239
+ self.projection_dim = projection_dim
240
+ self.num_hidden_layers = num_hidden_layers
241
+ self.num_attention_heads = num_attention_heads
242
+ self.num_channels = num_channels
243
+ self.patch_size = patch_size
244
+ self.image_size = image_size
245
+ self.initializer_range = initializer_range
246
+ self.initializer_factor = initializer_factor
247
+ self.attention_dropout = attention_dropout
248
+ self.layer_norm_eps = layer_norm_eps
249
+ self.hidden_act = hidden_act
250
+ self.use_flash_attn = use_flash_attn
251
+
252
+ @classmethod
253
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
254
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
255
+
256
+ # get the vision config dict if we are loading from MplugOwlConfig
257
+ if config_dict.get("model_type") == "mplug-owl":
258
+ config_dict = config_dict["vision_config"]
259
+
260
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
261
+ logger.warning(
262
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
263
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
264
+ )
265
+
266
+ return cls.from_dict(config_dict, **kwargs)
267
+
268
+
269
+ class MplugOwlVisualAbstractorConfig(PretrainedConfig):
270
+ model_type = "mplug_owl_visual_abstract"
271
+
272
+ def __init__(
273
+ self,
274
+ num_learnable_queries=64,
275
+ hidden_size=1024,
276
+ num_hidden_layers=6,
277
+ num_attention_heads=16,
278
+ intermediate_size=2816,
279
+ attention_probs_dropout_prob=0.,
280
+ initializer_range=0.02,
281
+ layer_norm_eps=1e-6,
282
+ encoder_hidden_size=1024,
283
+ grid_size=None,
284
+ **kwargs,
285
+ ):
286
+ super().__init__(**kwargs)
287
+ self.hidden_size = hidden_size
288
+ self.num_learnable_queries = num_learnable_queries
289
+ self.num_hidden_layers = num_hidden_layers
290
+ self.num_attention_heads = num_attention_heads
291
+ self.intermediate_size = intermediate_size
292
+ self.attention_probs_dropout_prob = attention_probs_dropout_prob
293
+ self.initializer_range = initializer_range
294
+ self.layer_norm_eps = layer_norm_eps
295
+ self.encoder_hidden_size = encoder_hidden_size
296
+ self.grid_size = grid_size if grid_size else 32
297
+
298
+ @classmethod
299
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
300
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
301
+
302
+ # get the visual_abstractor config dict if we are loading from MplugOwlConfig
303
+ if config_dict.get("model_type") == "mplug-owl":
304
+ config_dict = config_dict["abstractor_config"]
305
+
306
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
307
+ logger.warning(
308
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
309
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
310
+ )
311
+
312
+ return cls.from_dict(config_dict, **kwargs)
313
+
314
+
315
+
316
+ DEFAULT_VISUAL_CONFIG = {
317
+ "visual_model": MplugOwlVisionConfig().to_dict(),
318
+ "visual_abstractor": MplugOwlVisualAbstractorConfig().to_dict()
319
+ }
320
+
321
+ class MPLUGOwl2Config(LlamaConfig):
322
+ model_type = "mplug_owl2"
323
+ def __init__(self, visual_config=None, **kwargs):
324
+ if visual_config is None:
325
+ self.visual_config = DEFAULT_VISUAL_CONFIG
326
+ else:
327
+ self.visual_config = visual_config
328
+
329
+ super().__init__(
330
+ **kwargs,
331
+ )
332
+
333
+ if __name__ == "__main__":
334
+ print(MplugOwlVisionConfig().to_dict())