Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+sample-15s.wav filter=lfs diff=lfs merge=lfs -text

.yml

@@ -0,0 +1,9 @@
+speech_recognition:
+  waveform_path: "C:/Users/baby7/Desktop/multi_modal_model/sample-15s.wav"
+  task: "speech_recognition"
+  output_audio_key: "Transcription"
+
+tests:
+  - name: speech_recognition_test
+    waveform_path: "C:/Users/baby7/Desktop/multi_modal_model/sample-15s.wav"
+    expected_output: "Expected transcription"

SJMT_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b83992690213c479a00be25af7fa6bfea7526861094cff5b6e97ee44d89f1cbb
+size 2493181678

app.py

@@ -0,0 +1,25 @@
+def mock_text_generation(input_text):
+    # 模拟文本生成逻辑
+    if input_text == "Tell me a joke.":
+        return "Why don't scientists trust atoms? Because they make up everything!"
+    return "I can come up with many ideas, but that request has stumped me!"
+
+def mock_code_generation(input_code):
+    # 模拟代码生成逻辑
+    if input_code == "def greet(name):":
+        return "def greet(name):\n    return f'Hello, {name}!'"
+    return "Hmm, I'm not sure how to complete that one."
+
+# 测试文本生成功能
+input_text = "Tell me a joke."
+expected_output_text = "Why don't scientists trust atoms? Because they make up everything!"
+generated_text = mock_text_generation(input_text)
+assert generated_text == expected_output_text, f"Text generation test failed: expected {expected_output_text}, got {generated_text}"
+print("Text generation test passed")
+
+# 测试代码生成功能
+input_code = "def greet(name):"
+expected_output_code = "def greet(name):\n    return f'Hello, {name}!'"
+generated_code = mock_code_generation(input_code)
+assert generated_code == expected_output_code, f"Code generation test failed: expected {expected_output_code}, got {generated_code}"
+print("Code generation test passed")

code_generator/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

code_generator/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

config.json

@@ -0,0 +1,48 @@
+{
+    "model_name": "Evolutionary Multi-Modal Model",
+    "model_type": "transformer",
+    "license": "mit",
+    "language": "en",
+    "datasets": [
+      "Custom"
+    ],
+    "tags": [
+      "text-generation",
+      "code-generation",
+      "speech-recognition",
+      "multi-modal",
+      "evolutionary"
+    ],
+    "base_model": "facebook/bart-base",
+    "finetuned_from": [
+      "gpt2",
+      "bert-base-uncased",
+      "facebook/wav2vec2-base-960h",
+      "openai/clip-vit-base-patch32"
+    ],
+    "dataset": "Custom Multi-Modal Dataset",
+    "metrics": [
+      "perplexity",
+      "bleu",
+      "wer",
+      "cer"
+    ],
+    "library_name": "transformers",
+    "pipeline_tag": "text-generation",
+    "inference": {
+      "parameters": {
+        "max_length": 50,
+        "top_k": 50,
+        "top_p": 0.95,
+        "temperature": 1.2,
+        "do_sample": true
+      }
+    },
+    "extra_info": {
+      "author": "zero",
+      "version": "1.0",
+      "description":"1.0",
+      "citation": "@1.0"  
+    }
+  }
+  

config.yml

@@ -0,0 +1,73 @@
+model_name: Evolutionary Multi-Modal Model
+model_type: transformer
+license: mit
+language: en zh
+datasets:
+  - "Custom"
+tags:
+  - text-generation
+  - code-generation
+  - speech-recognition
+  - multi-modal
+  - evolutionary
+base_model: facebook/bart-base
+finetuned_from: gpt2, bert-base-uncased, facebook/wav2vec2-base-960h, openai/clip-vit-base-patch32
+dataset: Custom Multi-Modal Dataset
+
+metrics:
+  - perplexity
+  - bleu
+  - wer
+  - cer
+
+library_name: transformers
+pipeline_tag: text-generation
+inference:
+  parameters:
+    max_length: 50
+    top_k: 50
+    top_p: 0.95
+    temperature: 1.2
+    do_sample: true
+
+speech_recognition:
+  waveform_path: "C:/Users/baby7/Desktop/权重参数/sample-15s.wav"
+  task: "speech_recognition"
+  output_audio_key: "Transcription"
+
+text_generation:
+  input_text: "What is the future of AI?"
+  task: "text_generation"
+  output_text_key: "Generated Text"
+
+code_generation:
+  input_code: "def add(a, b): return"
+  task: "code_generation"
+  output_code_key: "Generated Code"
+
+tests:
+  - name: speech_recognition_test
+    waveform_path: "C:/Users/baby7/Desktop/权重参数/sample-15s.wav"
+    expected_output: "Expected transcription"
+  - name: text_generation_test
+    input_text: "What is the future of AI?"
+    expected_output: "Predicted text about AI"
+  - name: code_generation_test
+    input_code: "def add(a, b): return"
+    expected_output: "def add(a, b): return a + b"
+
+extra_info:
+  author: zero
+  version: 1.0
+  description: |
+    This Evolutionary Multi-Modal Model is designed for tasks like text generation, code generation, 
+    speech recognition, and vision understanding. It leverages the capabilities of multiple pre-trained
+    models and applies evolutionary techniques to optimize performance across these tasks.
+  citation:
+    - |
+      @article{your_reference_2025,
+        title={Evolutionary Multi-Modal Model for Enhanced Performance},
+        author={Your Name},
+        journal={Journal of AI Research},
+        year={2025}
+      }

nlp_encoder/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

nlp_encoder/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

sample-15s.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e2e7b2ffa021275a90a26704d923fe902d3600e4ffecf06253c57778a2a986
+size 3382316

speech_encoder/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

speech_encoder/tokenizer_config.json

@@ -0,0 +1,50 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "do_normalize": true,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "replace_word_delimiter_char": " ",
+  "return_attention_mask": false,
+  "target_lang": null,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "<unk>",
+  "word_delimiter_token": "|"
+}

speech_encoder/vocab.json

@@ -0,0 +1,34 @@
+{
+  "'": 27,
+  "</s>": 2,
+  "<pad>": 0,
+  "<s>": 1,
+  "<unk>": 3,
+  "A": 7,
+  "B": 24,
+  "C": 19,
+  "D": 14,
+  "E": 5,
+  "F": 20,
+  "G": 21,
+  "H": 11,
+  "I": 10,
+  "J": 29,
+  "K": 26,
+  "L": 15,
+  "M": 17,
+  "N": 9,
+  "O": 8,
+  "P": 23,
+  "Q": 30,
+  "R": 13,
+  "S": 12,
+  "T": 6,
+  "U": 16,
+  "V": 25,
+  "W": 18,
+  "X": 28,
+  "Y": 22,
+  "Z": 31,
+  "|": 4
+}

text_generator/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

text_generator/tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "BartTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

vision_encoder/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vision_encoder/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}