Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: vidore/colqwen2-base
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "vidore/colqwen2-base",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1b406ffda2f8a22c20ebe5faffc96293be6ed8b63b3a4235bf2e7133085115d
+size 73911504

added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

git_hash.txt

@@ -0,0 +1 @@
+72d566e2fd10750cb24dc45d4d5bddcb3a9366cc

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "BiQwen2Processor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "max_pixels": 12845056,
+    "min_pixels": 3136
+  },
+  "temporal_patch_size": 2
+}

results.json

@@ -0,0 +1 @@
+{"validation_set": {"ndcg_at_1": 0.716, "ndcg_at_3": 0.79, "ndcg_at_5": 0.80162, "ndcg_at_10": 0.81647, "ndcg_at_20": 0.82376, "ndcg_at_50": 0.8278, "ndcg_at_100": 0.8304, "map_at_1": 0.716, "map_at_3": 0.772, "map_at_5": 0.7785, "map_at_10": 0.78462, "map_at_20": 0.78673, "map_at_50": 0.78741, "map_at_100": 0.78764, "recall_at_1": 0.716, "recall_at_3": 0.842, "recall_at_5": 0.87, "recall_at_10": 0.916, "recall_at_20": 0.944, "recall_at_50": 0.964, "recall_at_100": 0.98, "precision_at_1": 0.716, "precision_at_3": 0.28067, "precision_at_5": 0.174, "precision_at_10": 0.0916, "precision_at_20": 0.0472, "precision_at_50": 0.01928, "precision_at_100": 0.0098, "mrr_at_1": 0.718, "mrr_at_3": 0.7746666666666666, "mrr_at_5": 0.7806666666666666, "mrr_at_10": 0.7868285714285714, "mrr_at_20": 0.7888180706118786, "mrr_at_50": 0.7895996953694904, "mrr_at_100": 0.7898277337976115, "naucs_at_1_max": 0.8001312345855925, "naucs_at_1_std": 0.11250348827562291, "naucs_at_1_diff1": 0.8830428322535393, "naucs_at_3_max": 0.9284596953407236, "naucs_at_3_std": 0.3631205849855362, "naucs_at_3_diff1": 0.8605631354827493, "naucs_at_5_max": 0.9320521365622464, "naucs_at_5_std": 0.4144190179960025, "naucs_at_5_diff1": 0.8710805006294897, "naucs_at_10_max": 0.9234138099684321, "naucs_at_10_std": 0.517418078342447, "naucs_at_10_diff1": 0.8689586945889484, "naucs_at_20_max": 0.9554821928771539, "naucs_at_20_std": 0.6497932506335875, "naucs_at_20_diff1": 0.921051754034949, "naucs_at_50_max": 0.9700435729847456, "naucs_at_50_std": 0.7666511048864019, "naucs_at_50_diff1": 0.9173928830791599, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 0.9460784313725422}, "syntheticDocQA_energy": {"ndcg_at_1": 0.75, "ndcg_at_3": 0.82678, "ndcg_at_5": 0.82678, "ndcg_at_10": 0.84683, "ndcg_at_20": 0.8521, "ndcg_at_50": 0.85785, "ndcg_at_100": 0.85785, "map_at_1": 0.75, "map_at_3": 0.80833, "map_at_5": 0.80833, "map_at_10": 0.81696, "map_at_20": 0.81851, "map_at_50": 0.81935, "map_at_100": 0.81935, "recall_at_1": 0.75, "recall_at_3": 0.88, "recall_at_5": 0.88, "recall_at_10": 0.94, "recall_at_20": 0.96, "recall_at_50": 0.99, "recall_at_100": 0.99, "precision_at_1": 0.75, "precision_at_3": 0.29333, "precision_at_5": 0.176, "precision_at_10": 0.094, "precision_at_20": 0.048, "precision_at_50": 0.0198, "precision_at_100": 0.0099, "mrr_at_1": 0.75, "mrr_at_3": 0.8083333333333332, "mrr_at_5": 0.8083333333333332, "mrr_at_10": 0.8167261904761905, "mrr_at_20": 0.8182738095238095, "mrr_at_50": 0.8191393489763056, "mrr_at_100": 0.8191393489763056, "naucs_at_1_max": 0.4352485609628468, "naucs_at_1_std": -0.3026268969126114, "naucs_at_1_diff1": 0.764772370486656, "naucs_at_3_max": 0.3731705377028315, "naucs_at_3_std": -0.43238943684377823, "naucs_at_3_diff1": 0.6174435252943048, "naucs_at_5_max": 0.37317053770283104, "naucs_at_5_std": -0.4323894368437778, "naucs_at_5_diff1": 0.6174435252943049, "naucs_at_10_max": 0.6125894802365398, "naucs_at_10_std": -0.20728291316526842, "naucs_at_10_diff1": 0.8031434796140697, "naucs_at_20_max": 0.5303454715219444, "naucs_at_20_std": -0.406629318394016, "naucs_at_20_diff1": 0.7047152194211033, "naucs_at_50_max": 0.8692810457516374, "naucs_at_50_std": 0.554154995331464, "naucs_at_50_diff1": 0.8692810457516374, "naucs_at_100_max": 0.8692810457516374, "naucs_at_100_std": 0.554154995331464, "naucs_at_100_diff1": 0.8692810457516374}, "syntheticDocQA_healthcare_industry": {"ndcg_at_1": 0.8, "ndcg_at_3": 0.87678, "ndcg_at_5": 0.89788, "ndcg_at_10": 0.90445, "ndcg_at_20": 0.90445, "ndcg_at_50": 0.90445, "ndcg_at_100": 0.90445, "map_at_1": 0.8, "map_at_3": 0.85833, "map_at_5": 0.87033, "map_at_10": 0.87311, "map_at_20": 0.87311, "map_at_50": 0.87311, "map_at_100": 0.87311, "recall_at_1": 0.8, "recall_at_3": 0.93, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.8, "precision_at_3": 0.31, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.8583333333333333, "mrr_at_5": 0.8703333333333333, "mrr_at_10": 0.8732500000000001, "mrr_at_20": 0.8732500000000001, "mrr_at_50": 0.8732500000000001, "mrr_at_100": 0.8732500000000001, "naucs_at_1_max": 0.32232767232767134, "naucs_at_1_std": -0.22445054945054957, "naucs_at_1_diff1": 0.6103646353646349, "naucs_at_3_max": 0.49893290649593114, "naucs_at_3_std": -0.6459250366813389, "naucs_at_3_diff1": 0.6492597038815514, "naucs_at_5_max": 0.6790382819794609, "naucs_at_5_std": -0.9556489262371534, "naucs_at_5_diff1": 0.861111111111116, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.71, "ndcg_at_3": 0.81071, "ndcg_at_5": 0.82276, "ndcg_at_10": 0.83243, "ndcg_at_20": 0.84238, "ndcg_at_50": 0.84624, "ndcg_at_100": 0.84624, "map_at_1": 0.71, "map_at_3": 0.78667, "map_at_5": 0.79317, "map_at_10": 0.79713, "map_at_20": 0.79978, "map_at_50": 0.80035, "map_at_100": 0.80035, "recall_at_1": 0.71, "recall_at_3": 0.88, "recall_at_5": 0.91, "recall_at_10": 0.94, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.71, "precision_at_3": 0.29333, "precision_at_5": 0.182, "precision_at_10": 0.094, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.71, "mrr_at_3": 0.7866666666666667, "mrr_at_5": 0.7931666666666666, "mrr_at_10": 0.7971349206349206, "mrr_at_20": 0.7998128023936847, "mrr_at_50": 0.8003853830388461, "mrr_at_100": 0.8003853830388461, "naucs_at_1_max": 0.4346867297963423, "naucs_at_1_std": -0.26896206660118943, "naucs_at_1_diff1": 0.8395567333172729, "naucs_at_3_max": 0.5740136811963086, "naucs_at_3_std": -0.24773305758829017, "naucs_at_3_diff1": 0.7142857142857145, "naucs_at_5_max": 0.6865338728083817, "naucs_at_5_std": -0.06338831829027824, "naucs_at_5_diff1": 0.7004876024483871, "naucs_at_10_max": 0.8712262682850935, "naucs_at_10_std": 0.24198568316215255, "naucs_at_10_diff1": 0.6896202925614692, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": 0.5613912231559286, "naucs_at_20_diff1": 0.861111111111116, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "syntheticDocQA_government_reports": {"ndcg_at_1": 0.7, "ndcg_at_3": 0.80202, "ndcg_at_5": 0.83042, "ndcg_at_10": 0.84047, "ndcg_at_20": 0.84812, "ndcg_at_50": 0.84812, "ndcg_at_100": 0.84812, "map_at_1": 0.7, "map_at_3": 0.77833, "map_at_5": 0.79383, "map_at_10": 0.79818, "map_at_20": 0.80031, "map_at_50": 0.80031, "map_at_100": 0.80031, "recall_at_1": 0.7, "recall_at_3": 0.87, "recall_at_5": 0.94, "recall_at_10": 0.97, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.7, "precision_at_3": 0.29, "precision_at_5": 0.188, "precision_at_10": 0.097, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.7, "mrr_at_3": 0.7783333333333333, "mrr_at_5": 0.7938333333333334, "mrr_at_10": 0.7981785714285714, "mrr_at_20": 0.8003052682611507, "mrr_at_50": 0.8003052682611507, "mrr_at_100": 0.8003052682611507, "naucs_at_1_max": 0.34667627454512717, "naucs_at_1_std": -0.18418303008467002, "naucs_at_1_diff1": 0.7120158529994597, "naucs_at_3_max": 0.4705991261201206, "naucs_at_3_std": -0.35103310375472296, "naucs_at_3_diff1": 0.44619714137598854, "naucs_at_5_max": 0.6545284780578933, "naucs_at_5_std": -0.7740429505135397, "naucs_at_5_diff1": 0.6678338001867429, "naucs_at_10_max": 0.5938375350140046, "naucs_at_10_std": -0.7231559290382794, "naucs_at_10_diff1": 0.6204481792717093, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "infovqa_subsampled": {"ndcg_at_1": 0.692, "ndcg_at_3": 0.76431, "ndcg_at_5": 0.78548, "ndcg_at_10": 0.80119, "ndcg_at_20": 0.80674, "ndcg_at_50": 0.81252, "ndcg_at_100": 0.81415, "map_at_1": 0.692, "map_at_3": 0.74767, "map_at_5": 0.75927, "map_at_10": 0.76587, "map_at_20": 0.76738, "map_at_50": 0.7684, "map_at_100": 0.76855, "recall_at_1": 0.692, "recall_at_3": 0.812, "recall_at_5": 0.864, "recall_at_10": 0.912, "recall_at_20": 0.934, "recall_at_50": 0.962, "recall_at_100": 0.972, "precision_at_1": 0.692, "precision_at_3": 0.27067, "precision_at_5": 0.1728, "precision_at_10": 0.0912, "precision_at_20": 0.0467, "precision_at_50": 0.01924, "precision_at_100": 0.00972, "mrr_at_1": 0.69, "mrr_at_3": 0.7473333333333334, "mrr_at_5": 0.7589333333333332, "mrr_at_10": 0.7651642857142857, "mrr_at_20": 0.7666584847299244, "mrr_at_50": 0.7676865192114355, "mrr_at_100": 0.7678320500103497, "naucs_at_1_max": 0.3834175233264141, "naucs_at_1_std": -0.27314752495589745, "naucs_at_1_diff1": 0.8045292563177913, "naucs_at_3_max": 0.577343039126478, "naucs_at_3_std": -0.008935184205513267, "naucs_at_3_diff1": 0.7278181010220708, "naucs_at_5_max": 0.675093872674517, "naucs_at_5_std": 0.18129515844569502, "naucs_at_5_diff1": 0.7001977015417865, "naucs_at_10_max": 0.8044414735591211, "naucs_at_10_std": 0.42251294457176947, "naucs_at_10_diff1": 0.7054579407520588, "naucs_at_20_max": 0.8796932914579985, "naucs_at_20_std": 0.6807854454913284, "naucs_at_20_diff1": 0.7039583510171761, "naucs_at_50_max": 0.9501203990368081, "naucs_at_50_std": 0.857265713302859, "naucs_at_50_diff1": 0.7446311858076538, "naucs_at_100_max": 0.9416433239962615, "naucs_at_100_std": 0.8673135921035019, "naucs_at_100_diff1": 0.6826063758836863}, "docvqa_subsampled": {"ndcg_at_1": 0.344, "ndcg_at_3": 0.41347, "ndcg_at_5": 0.44369, "ndcg_at_10": 0.46201, "ndcg_at_20": 0.4759, "ndcg_at_50": 0.4878, "ndcg_at_100": 0.5002, "map_at_1": 0.344, "map_at_3": 0.39667, "map_at_5": 0.41377, "map_at_10": 0.42144, "map_at_20": 0.42512, "map_at_50": 0.42702, "map_at_100": 0.42814, "recall_at_1": 0.344, "recall_at_3": 0.462, "recall_at_5": 0.534, "recall_at_10": 0.59, "recall_at_20": 0.646, "recall_at_50": 0.706, "recall_at_100": 0.782, "precision_at_1": 0.344, "precision_at_3": 0.154, "precision_at_5": 0.1068, "precision_at_10": 0.059, "precision_at_20": 0.0323, "precision_at_50": 0.01412, "precision_at_100": 0.00782, "mrr_at_1": 0.346, "mrr_at_3": 0.398, "mrr_at_5": 0.4155, "mrr_at_10": 0.42276111111111114, "mrr_at_20": 0.426310462043976, "mrr_at_50": 0.42830398638968425, "mrr_at_100": 0.4294428772908946, "naucs_at_1_max": 0.7371100485555001, "naucs_at_1_std": 0.33261575567803164, "naucs_at_1_diff1": 0.8130375220827029, "naucs_at_3_max": 0.8020023826617959, "naucs_at_3_std": 0.49953582802757135, "naucs_at_3_diff1": 0.7334337943494619, "naucs_at_5_max": 0.8117119579471634, "naucs_at_5_std": 0.5755264196835543, "naucs_at_5_diff1": 0.6923091789040089, "naucs_at_10_max": 0.8227097304356757, "naucs_at_10_std": 0.613846537631884, "naucs_at_10_diff1": 0.7031446725461138, "naucs_at_20_max": 0.8705519074674019, "naucs_at_20_std": 0.7222923846653254, "naucs_at_20_diff1": 0.7266076440815379, "naucs_at_50_max": 0.8427827590881949, "naucs_at_50_std": 0.7425625391614057, "naucs_at_50_diff1": 0.6988519982851428, "naucs_at_100_max": 0.8260205582772512, "naucs_at_100_std": 0.7179798527014848, "naucs_at_100_diff1": 0.683312862233389}, "arxivqa_subsampled": {"ndcg_at_1": 0.714, "ndcg_at_3": 0.77369, "ndcg_at_5": 0.79013, "ndcg_at_10": 0.79976, "ndcg_at_20": 0.8069, "ndcg_at_50": 0.8186, "ndcg_at_100": 0.82249, "map_at_1": 0.714, "map_at_3": 0.75967, "map_at_5": 0.76877, "map_at_10": 0.7727, "map_at_20": 0.77469, "map_at_50": 0.77682, "map_at_100": 0.77716, "recall_at_1": 0.714, "recall_at_3": 0.814, "recall_at_5": 0.854, "recall_at_10": 0.884, "recall_at_20": 0.912, "recall_at_50": 0.968, "recall_at_100": 0.992, "precision_at_1": 0.714, "precision_at_3": 0.27133, "precision_at_5": 0.1708, "precision_at_10": 0.0884, "precision_at_20": 0.0456, "precision_at_50": 0.01936, "precision_at_100": 0.00992, "mrr_at_1": 0.712, "mrr_at_3": 0.7586666666666666, "mrr_at_5": 0.7673666666666666, "mrr_at_10": 0.771681746031746, "mrr_at_20": 0.7736755779308411, "mrr_at_50": 0.7757993229634467, "mrr_at_100": 0.7761433854912912, "naucs_at_1_max": 0.7317309098271223, "naucs_at_1_std": -0.37215292384176624, "naucs_at_1_diff1": 0.8526437738310815, "naucs_at_3_max": 0.8049699197503837, "naucs_at_3_std": -0.23096075141852557, "naucs_at_3_diff1": 0.8225158415524194, "naucs_at_5_max": 0.8093326040846809, "naucs_at_5_std": -0.24028520736897105, "naucs_at_5_diff1": 0.7992676387017601, "naucs_at_10_max": 0.8183301091839064, "naucs_at_10_std": -0.2813370930194455, "naucs_at_10_diff1": 0.7861733171579385, "naucs_at_20_max": 0.8187866055513119, "naucs_at_20_std": -0.1553348612172151, "naucs_at_20_diff1": 0.7767910194380784, "naucs_at_50_max": 0.873511904761906, "naucs_at_50_std": 0.2547852474323104, "naucs_at_50_diff1": 0.7219887955182044, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.9673202614378978, "naucs_at_100_diff1": 0.8978758169934562}, "tabfquad_subsampled": {"ndcg_at_1": 0.74286, "ndcg_at_3": 0.79235, "ndcg_at_5": 0.81125, "ndcg_at_10": 0.82042, "ndcg_at_20": 0.83173, "ndcg_at_50": 0.83813, "ndcg_at_100": 0.84176, "map_at_1": 0.74286, "map_at_3": 0.78095, "map_at_5": 0.79131, "map_at_10": 0.79507, "map_at_20": 0.79795, "map_at_50": 0.79898, "map_at_100": 0.79935, "recall_at_1": 0.74286, "recall_at_3": 0.825, "recall_at_5": 0.87143, "recall_at_10": 0.9, "recall_at_20": 0.94643, "recall_at_50": 0.97857, "recall_at_100": 1.0, "precision_at_1": 0.74286, "precision_at_3": 0.275, "precision_at_5": 0.17429, "precision_at_10": 0.09, "precision_at_20": 0.04732, "precision_at_50": 0.01957, "precision_at_100": 0.01, "mrr_at_1": 0.7428571428571429, "mrr_at_3": 0.7803571428571429, "mrr_at_5": 0.7916071428571428, "mrr_at_10": 0.7947760770975056, "mrr_at_20": 0.7976513639285919, "mrr_at_50": 0.7986905677649363, "mrr_at_100": 0.799055515734078, "naucs_at_1_max": 0.7500073065232634, "naucs_at_1_std": 0.2514466916062664, "naucs_at_1_diff1": 0.8531827215337847, "naucs_at_3_max": 0.8343426391328044, "naucs_at_3_std": 0.4195381416661603, "naucs_at_3_diff1": 0.8111084196289781, "naucs_at_5_max": 0.8508788332086781, "naucs_at_5_std": 0.5882973608291492, "naucs_at_5_diff1": 0.8132145528368421, "naucs_at_10_max": 0.8186774709883973, "naucs_at_10_std": 0.5969054288382045, "naucs_at_10_diff1": 0.7904995331465913, "naucs_at_20_max": 0.8555866791160895, "naucs_at_20_std": 0.659228135698729, "naucs_at_20_diff1": 0.7743541861188911, "naucs_at_50_max": 0.7786336756925044, "naucs_at_50_std": 0.438297541238722, "naucs_at_50_diff1": 0.6545284780578803, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "tatdqa": {"ndcg_at_1": 0.44197, "ndcg_at_3": 0.55566, "ndcg_at_5": 0.58437, "ndcg_at_10": 0.62411, "ndcg_at_20": 0.64062, "ndcg_at_50": 0.65097, "ndcg_at_100": 0.65588, "map_at_1": 0.44197, "map_at_3": 0.52726, "map_at_5": 0.54304, "map_at_10": 0.55982, "map_at_20": 0.56448, "map_at_50": 0.56618, "map_at_100": 0.56662, "recall_at_1": 0.44197, "recall_at_3": 0.638, "recall_at_5": 0.70836, "recall_at_10": 0.82922, "recall_at_20": 0.89357, "recall_at_50": 0.94528, "recall_at_100": 0.97535, "precision_at_1": 0.44197, "precision_at_3": 0.21267, "precision_at_5": 0.14167, "precision_at_10": 0.08292, "precision_at_20": 0.04468, "precision_at_50": 0.01891, "precision_at_100": 0.00975, "mrr_at_1": 0.4395670475045099, "mrr_at_3": 0.525756664662257, "mrr_at_5": 0.5415714572058529, "mrr_at_10": 0.5584664834063511, "mrr_at_20": 0.5630899072695948, "mrr_at_50": 0.5647839013198243, "mrr_at_100": 0.5652361863084263, "naucs_at_1_max": 0.38576514235208315, "naucs_at_1_std": -0.08015654973630132, "naucs_at_1_diff1": 0.603618573280768, "naucs_at_3_max": 0.43656957234090016, "naucs_at_3_std": -0.021403883055059796, "naucs_at_3_diff1": 0.47048780566558884, "naucs_at_5_max": 0.4336016828393664, "naucs_at_5_std": -0.0005891945007471499, "naucs_at_5_diff1": 0.4256629282001986, "naucs_at_10_max": 0.5493338687113005, "naucs_at_10_std": 0.16083596394909894, "naucs_at_10_diff1": 0.38247082357061657, "naucs_at_20_max": 0.5584090277396198, "naucs_at_20_std": 0.2602657817525008, "naucs_at_20_diff1": 0.3980322374475721, "naucs_at_50_max": 0.633233633128742, "naucs_at_50_std": 0.4310547495591043, "naucs_at_50_diff1": 0.38621143845546824, "naucs_at_100_max": 0.8349677169227036, "naucs_at_100_std": 0.7325722103484296, "naucs_at_100_diff1": 0.4877781738863079}, "shift_project": {"ndcg_at_1": 0.52, "ndcg_at_3": 0.63702, "ndcg_at_5": 0.6779, "ndcg_at_10": 0.7029, "ndcg_at_20": 0.71283, "ndcg_at_50": 0.72072, "ndcg_at_100": 0.72246, "map_at_1": 0.52, "map_at_3": 0.60833, "map_at_5": 0.63083, "map_at_10": 0.64065, "map_at_20": 0.64327, "map_at_50": 0.64451, "map_at_100": 0.6447, "recall_at_1": 0.52, "recall_at_3": 0.72, "recall_at_5": 0.82, "recall_at_10": 0.9, "recall_at_20": 0.94, "recall_at_50": 0.98, "recall_at_100": 0.99, "precision_at_1": 0.52, "precision_at_3": 0.24, "precision_at_5": 0.164, "precision_at_10": 0.09, "precision_at_20": 0.047, "precision_at_50": 0.0196, "precision_at_100": 0.0099, "mrr_at_1": 0.55, "mrr_at_3": 0.6216666666666666, "mrr_at_5": 0.6451666666666667, "mrr_at_10": 0.6549841269841269, "mrr_at_20": 0.6576700244200243, "mrr_at_50": 0.6589199025112817, "mrr_at_100": 0.6591122102035896, "naucs_at_1_max": 0.05023396267479185, "naucs_at_1_std": -0.2630855868110345, "naucs_at_1_diff1": 0.5195467653299963, "naucs_at_3_max": 0.2670177045177047, "naucs_at_3_std": -0.04368894993894898, "naucs_at_3_diff1": 0.3609394078144081, "naucs_at_5_max": 0.5939530785328778, "naucs_at_5_std": 0.2729926203326362, "naucs_at_5_diff1": 0.2156625178984482, "naucs_at_10_max": 0.6814192343604119, "naucs_at_10_std": 0.3050420168067224, "naucs_at_10_diff1": 0.08445378151260835, "naucs_at_20_max": 0.7568471833177721, "naucs_at_20_std": 0.39285714285714246, "naucs_at_20_diff1": 0.20028011204481636, "naucs_at_50_max": 0.5401493930905731, "naucs_at_50_std": -0.17133520074695377, "naucs_at_50_diff1": 0.4960317460317565, "naucs_at_100_max": 0.35807656395892007, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.12278244631185525}}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091aa7594dc2fcfbfa06b9e3c22a5f0562ac14f30375c13af7309407a0e67b8a
+size 11420371

tokenizer_config.json

@@ -0,0 +1,144 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "processor_class": "BiQwen2Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_config.yml

@@ -0,0 +1,67 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/biqwen2-1e-256-newpad-nobuffer
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2Processor
+    pretrained_model_name_or_path:  "./models/colqwen2_base" # "./models/paligemma-3b-mix-448"
+    # max_length: 50
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2
+    pretrained_model_name_or_path: "./models/colqwen2_base"
+    torch_dtype:  !ext torch.bfloat16
+    use_cache: false
+    attn_implementation: "flash_attention_2"
+  #    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  add_suffix: true
+  loss_func:
+    (): colpali_engine.loss.bi_encoder_losses.BiPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 1
+    per_device_train_batch_size: 64
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 64
+    eval_strategy: "steps"
+    dataloader_num_workers: 8
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-4
+    save_total_limit: 1
+    resume_from_checkpoint: false
+    report_to: "wandb"
+
+    # optim: "paged_adamw_8bit"
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+