Upload 24 files

.gitattributes

@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+show_case/audio-1434542201-headset.wav filter=lfs diff=lfs merge=lfs -text
+show_case/p225_002.wav filter=lfs diff=lfs merge=lfs -text
+show_case/Real.wav filter=lfs diff=lfs merge=lfs -text
+show_case/Scene_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SER(emotion)_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SFT_Fisher_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SG_audio_1.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SGR_018.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SLR_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SNV_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/Sound_Vocal_example.wav filter=lfs diff=lfs merge=lfs -text
+show_case/SVD_14154_file31512.mp3.wav_16k.wav_norm.wav_mono.wav_silence.wav filter=lfs diff=lfs merge=lfs -text
+Soundwave/assets/audio/example_1.wav filter=lfs diff=lfs merge=lfs -text
+Soundwave/assets/logo.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -10,5 +10,5 @@ pinned: false
 license: apache-2.0
 short_description: The Official Demo of Soundwave
 ---
-
+This space is designed to provide an intuitive demonstration for the paper titled "Soundwave" and the link to the paper is: arxiv.org/abs/2502.12900.
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Soundwave/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Soundwave/README.md

@@ -0,0 +1,107 @@
+# Soundwave: *Less is More* for Speech-Text Alignment in LLMs
+
+<p align="center">
+  <img src="assets/logo.png" style="width:240px; height:240px; margin-bottom:10px;"/>
+</p>
+
+<p align="center">
+  <font size="3"><a href="https://huggingface.co/papers/2502.12900">🤗 Paper</a>&nbsp｜&nbsp<a href="https://huggingface.co/FreedomIntelligence/Soundwave">🤗 Model</a>｜&nbsp<a href="https://arxiv.org/abs/2502.12900">📃 Paper</a>｜&nbsp<a href="https://huggingface.co/spaces/FreedomIntelligence/SoundwaveDemo">📼 Online Demo</a>&nbsp</font>
+</p>
+
+<div>
+  <h2>✨ Highlights of Our Soundwave Model !️</h2>
+  <ul>
+    <font size="3"><li>A Speech-to-Text Model Bridging the Gap Between Speech and Text</li></font>
+    <font size="3"><li>Utilizes Data-Efficient Strategy and Unique Architecture, Trained on Only 10k Hours of Data</li></font>
+    <font size="3"><li>Exceptional Performance in Speech Translation and AIR-Bench Speech Tasks</li></font>
+    <font size="3"><li>Retains Intelligence During Conversations, Ideal for Interactive Tasks</li></font>
+  </ul>
+</div>
+
+
+## 💌 News
+> <ul>
+>   <font size="3"><li>[05/03/2025] 🔥 We released our Soundwave weights <a href="https://huggingface.co/FreedomIntelligence/Soundwave">🤗 Model </a> ! </li></font>
+>   <font size="3"><li>[19/02/2025] Try our model now in the <a href="https://huggingface.co/spaces/FreedomIntelligence/SoundwaveDemo">📼 Online Demo</a> . </li></font>
+>   <font size="3"><li>[19/02/2025] The online demo and model weights are coming soon. </li></font>
+>   <font size="3"><li>[18/02/2025] Release the model architecture and inference code. </li></font>
+> </ul>
+
+## Project Structure
+```
+.
+├── assets/
+│   └── audio/                     # Directory for test audio files (e.g., .wav files)
+├── README.md                      
+├── run_inference.py               # Main inference script
+└── Soundwave.py                   # Model architecture
+```
+
+
+## Getting Started
+
+### Installation Requirements
+<font size="3">Python version 3.10.11 is used in the Soundwave project.</font>
+```bash
+conda create -n soundwave python=3.10.11
+conda activate soundwave
+pip install -r requirements.txt 
+```
+
+## Inference
+> <font size="3">Before starting, ensure you have at least 21GB of GPU memory to run our model inference.</font><br>
+
+### Usage Command
+<font size="3">To run the inference script and process the audio, use the following command:</font>
+```bash
+python run_inference.py --model_path <model_path>
+# model_path: Path to the pre-trained Soundwave model.
+```
+
+<font size="3">Below are some quick usage examples you can try:</font>
+```python
+import torch
+import librosa
+from run_inference import load_model, gen_model_inputs, CONFIG
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+model, audio_processor, tokenizer = load_model("FreedomIntelligence/Soundwave", device)
+
+# apply chat template
+prompt = "What does the person say?"
+model_inputs = gen_model_inputs(tokenizer, prompt, device)
+
+ # audio preprocess
+audio_path = "assets/audio/example_1.wav"
+audio, _ = librosa.load(audio_path, sr=CONFIG.sampling_rate, mono=True)
+audio_feat = audio_processor(
+    audio, sampling_rate=CONFIG.sampling_rate, return_tensors="pt"
+).input_features.to(device, dtype=torch.float16)
+
+ # inference
+output_ids = model.generate(
+    **model_inputs,
+    audios=audio_feat,
+    max_new_tokens=512,
+    eos_token_id=tokenizer.eos_token_id,
+    do_sample=True,
+    top_p=0.9,
+    temperature=0.2
+)
+
+input_token_len = model_inputs["input_ids"].shape[1]
+response = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+
+print(response)
+```
+## Citation
+<font size="3">If you found this repository useful, please consider citing this work:</font>
+```
+@article{zhang2025soundwave,
+  title={Soundwave: Less is More for Speech-Text Alignment in LLMs},
+  author={Zhang, Yuhao and Liu, Zhiheng and Bu, Fan and Zhang, Ruiyu and Wang, Benyou and Li, Haizhou},
+  journal={arXiv preprint arXiv:2502.12900},
+  year={2025}
+}
+```

Soundwave/Soundwave.py

@@ -0,0 +1,341 @@
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn import CrossEntropyLoss
+
+
+from transformers import AutoConfig, AutoModelForCausalLM, \
+                         LlamaConfig, LlamaModel, LlamaForCausalLM
+from transformers.trainer_pt_utils import LabelSmoother
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from transformers.models.whisper.modeling_whisper import WhisperEncoder, WhisperConfig
+
+
+IGNORE_TOKEN_ID = LabelSmoother.ignore_index
+
+
+class SoundwaveConfig(LlamaConfig):
+    model_type = "Soundwave"
+    
+class LookBackModule(nn.Module):
+    def __init__(self, cfg: LlamaConfig):
+        super().__init__()
+        self.encoder_attn = nn.MultiheadAttention(
+            cfg.hidden_size,
+            cfg.num_attention_heads,
+            dropout=0.1,
+            batch_first=True
+        )
+        self.atten_layer_norm = nn.LayerNorm(cfg.hidden_size)
+
+
+    def forward(self, x, wav_feature, bf_shrink_padding_mask):
+
+        residual = x
+        x, _ = self.encoder_attn(
+            query=x,
+            key=wav_feature,
+            value=wav_feature,
+            key_padding_mask=bf_shrink_padding_mask,
+        )
+        x += residual
+        x = self.atten_layer_norm(x)
+        return x
+
+class SoundwaveModel(LlamaModel):
+    config_class = SoundwaveConfig
+
+    def __init__(self, config: LlamaConfig):
+        super(SoundwaveModel, self).__init__(config)
+
+        if hasattr(config, "adapter_size"):
+            self.mm_projector1 = nn.Linear(config.adapter_size*2 , config.hidden_size)
+            self.lbm =  LookBackModule(config)
+            self.out_norm = nn.LayerNorm(config.hidden_size)
+            self.audio_feature_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+            
+            asr_encoder_layer = nn.TransformerEncoderLayer(
+                d_model=config.hidden_size,
+                nhead=config.num_attention_heads,
+                dim_feedforward=config.hidden_size*2,
+                dropout=0.1,
+                norm_first=True
+            )
+            self.asr_transformer_encoder = nn.TransformerEncoder(asr_encoder_layer, num_layers=1)
+        
+        if hasattr(config, "audio_tower"):
+            self.audio_tower = WhisperEncoder(WhisperConfig.from_pretrained(config.audio_tower))
+        self.mask_tensor=(torch.ones([1,1024])>0)
+        self.length=-1
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        audios: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if (input_ids.shape[1] != 1 or self.training) and audios is not None:
+            audio_list=[]
+            
+            for audio in audios:
+                with torch.no_grad():
+                    audio=audio.unsqueeze(0)
+                    audio_feature = self.audio_tower(audio).last_hidden_state
+           
+                audio_feature = audio_feature.view(audio_feature.shape[0], audio_feature.shape[1]//2, 2 * audio_feature.shape[2])
+                audio_feature = self.mm_projector1(audio_feature)
+                audio_feature = self.asr_transformer_encoder(audio_feature)
+                audio_feature = self.out_norm(audio_feature)
+                audio_list.append(audio_feature[0])
+
+            audio_features = torch.stack(audio_list, dim=0)
+
+            predict_logits = self.audio_feature_head(audio_features)
+
+            new_input_embeds = []
+            label_shift = []
+            label_extend = -1
+            new_input_ids = []
+            tokens = predict_logits.argmax(dim=-1)
+            shrink_mask = tokens.roll(1) != tokens
+            shrink_mask[:,0] = True
+                
+            lengths = shrink_mask.long().sum(-1)
+            shrink_2d = audio_features[shrink_mask]
+            num_patches = self.config.audio_patch_size
+            l_index=0
+            shrink_features = []
+            for v, audio_feature, mask in zip(lengths, audio_features, ~shrink_mask):
+                shrink_feature = shrink_2d[l_index:l_index+v]
+                shrink_feature = self.lbm(shrink_feature, audio_feature, bf_shrink_padding_mask=mask)
+                shrink_features.append(shrink_feature)
+                l_index += v
+            
+            if self.training: 
+                maxn_length = lengths.max()
+                label_extend = maxn_length - num_patches
+                for cur_input_ids, cur_input_embeds, shrink_feature in zip(input_ids, inputs_embeds, shrink_features):
+                    pad_ids = torch.full(size=(maxn_length,), fill_value=self.config.llm_pad_token_id, dtype=torch.long).to(attention_mask.device)
+                    pad_embeds = self.embed_tokens(pad_ids)
+                    v = shrink_feature.shape[0]
+                    audio_start_token_pos = torch.where(cur_input_ids == self.config.audio_patch_token)[0][:1]
+                    cur_new_input_id = torch.cat((cur_input_ids[:audio_start_token_pos], cur_input_ids[audio_start_token_pos: audio_start_token_pos+1].repeat(v), cur_input_ids[audio_start_token_pos + num_patches:], pad_ids[:maxn_length - v]), dim=0)
+                    cur_new_input_embeds = torch.cat((
+                    cur_input_embeds[:audio_start_token_pos],
+                    shrink_feature,
+                    cur_input_embeds[audio_start_token_pos + num_patches:],pad_embeds[:maxn_length-v]), dim=0)
+                    new_input_embeds.append(cur_new_input_embeds)
+                    new_input_ids.append(cur_new_input_id)
+                    label_shift.append(v - num_patches)
+                    
+                input_ids = torch.stack(new_input_ids, dim=0)
+                attention_mask=input_ids.ne(self.config.llm_pad_token_id)
+                inputs_embeds = torch.stack(new_input_embeds, dim=0) 
+            else:
+                for cur_input_ids, cur_input_embeds, shrink_feature in zip(input_ids, inputs_embeds, shrink_features):
+                    v = shrink_feature.shape[0]
+
+                    audio_start_token_pos = torch.where(cur_input_ids == self.config.audio_patch_token)[0][:1]
+                    cur_new_input_id = torch.cat((cur_input_ids[:audio_start_token_pos],cur_input_ids[audio_start_token_pos: audio_start_token_pos+1].repeat(v), cur_input_ids[audio_start_token_pos + num_patches:]),dim=0)
+                    cur_new_input_embeds = torch.cat((
+                    cur_input_embeds[:audio_start_token_pos],
+                    shrink_feature,
+                    cur_input_embeds[audio_start_token_pos + num_patches:]), dim=0)
+                    new_input_embeds.append(cur_new_input_embeds)
+                    new_input_ids.append(cur_new_input_id)
+                input_ids = torch.stack(new_input_ids, dim=0)
+                attention_mask=input_ids.ne(self.config.llm_pad_token_id)
+                inputs_embeds = torch.stack(new_input_embeds, dim=0)
+                self.mask_tensor.to(input_ids.device)[0][:attention_mask.shape[1]]=attention_mask[0]
+                self.length=attention_mask.shape[1]
+
+        if not self.training:
+            attention_mask=self.mask_tensor.to(input_ids.device)[:,:self.length]
+            self.length+=1
+
+        return_state=super(SoundwaveModel, self).forward(
+            input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds, use_cache=use_cache,
+            output_attentions=output_attentions, output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+        if self.training:
+            return_state["audio_features"] = predict_logits
+            return_state["label_shift"] = label_shift
+            return_state["label_extend"] = label_extend
+        
+        return return_state 
+
+
+class SoundwaveForCausalLM(LlamaForCausalLM):
+    config_class = SoundwaveConfig
+
+    def __init__(self, config):
+        super(LlamaForCausalLM, self).__init__(config)
+        self.model = SoundwaveModel(config)
+
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        asr_targets: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        audios: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            audios=audios
+        )
+        
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            if asr_targets is not None:
+                mask_asr_targets = (asr_targets != IGNORE_TOKEN_ID)
+                target_lengths = mask_asr_targets.sum(1)
+                input_lengths = torch.full(size=(outputs["audio_features"].shape[0],), fill_value=outputs["audio_features"].shape[1], dtype=torch.long)
+                asr_logits = outputs["audio_features"]
+
+                log_probs = F.log_softmax(asr_logits, dim=-1).transpose(0, 1)
+
+                with torch.backends.cudnn.flags(enabled=False):
+                    loss_asr = F.ctc_loss(
+                        log_probs,
+                        asr_targets,
+                        input_lengths,
+                        target_lengths,
+                        blank=self.model.config.audio_patch_token,
+                        reduction='mean',
+                        zero_infinity=True,
+                    )
+            else:
+                loss_asr=0
+
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+
+            if len(outputs["label_shift"]) >0:
+                if outputs["label_extend"] != -1:
+                    new_shift_labels = torch.full(size=(shift_labels.shape[0], outputs["label_extend"]+shift_labels.shape[1]), fill_value=IGNORE_TOKEN_ID, dtype=torch.long).to(shift_labels.device)
+                    for i in range(len(outputs["label_shift"])):
+                        new_shift_labels[i][outputs["label_shift"][i]:outputs["label_shift"][i] + len(shift_labels[i])]= shift_labels[i]
+                    shift_labels = new_shift_labels
+                else:
+                    for i in range(len(outputs["label_shift"])):
+                        shift_labels[i]= shift_labels[i].roll(-outputs["label_shift"][i])
+
+            loss_fct = CrossEntropyLoss()
+            # Flatten the tokens
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+                    
+            # Enable model/pipeline parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+            loss = loss + 0.3*loss_asr 
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        use_cache=True,
+        **kwargs,
+    ):
+        # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens
+        # Exception 1: when passing input_embeds, input_ids may be missing entries
+        # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here
+        if past_key_values is not None:
+            if inputs_embeds is not None:  # Exception 1
+                input_ids = input_ids[:, -cache_position.shape[0] :]
+            elif input_ids.shape[1] != cache_position.shape[0]:  # Default case (the "else", a no op, is Exception 2)
+                input_ids = input_ids[:, cache_position]
+
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and cache_position[0] == 0:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids.contiguous()}  # `contiguous()` needed for compilation use cases
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "cache_position": cache_position,
+                "past_key_values": past_key_values,
+                "use_cache": use_cache,
+                "attention_mask": attention_mask,
+            }
+        )
+        model_inputs.update({"audios": kwargs["audios"]} if "audios" in kwargs.keys() else {})
+        return model_inputs
+
+
+AutoConfig.register("Soundwave", SoundwaveConfig)
+AutoModelForCausalLM.register(SoundwaveConfig, SoundwaveForCausalLM)

Soundwave/assets/audio/example_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f017d446358e8f9a0bd5d2cf209a3fa2c0e40025e2831142d591945b65fcd809
+size 137166

Soundwave/requirement.txt

@@ -0,0 +1,5 @@
+torch==2.3.0
+gradio
+librosa==0.10.2.post1
+transformers==4.43.1
+accelerate==0.34.2

Soundwave/run_inference.py

@@ -0,0 +1,123 @@
+import torch
+import argparse
+import librosa
+from transformers import AutoTokenizer, WhisperProcessor
+from .Soundwave import SoundwaveForCausalLM
+import spaces
+import gradio as gr
+
+class BasicSetting:
+    def __init__(self):
+        self.sampling_rate = 16000
+        self.audio_token_len = 1 
+        self.stop = "</s>"
+CONFIG = BasicSetting()
+
+
+def load_model(model_path, device):
+    # load model
+    model = SoundwaveForCausalLM.from_pretrained(
+        model_path,
+        device_map={"": device},
+        torch_dtype=torch.float16,
+        quantization_config=None,
+        # attn_implementation="flash_attention_2"
+    ).eval().to(device)
+    
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+    model.config.audio_patch_token = tokenizer.get_vocab()["<audio_patch>"]
+    model.config.llm_pad_token_id = tokenizer.pad_token_id
+    model.generation_config.pad_token_id = tokenizer.eos_token_id
+
+    # load audio preprocessor
+    audio_processor = WhisperProcessor.from_pretrained(model.config.audio_tower, torch_dtype=torch.float16)
+    return model, audio_processor, tokenizer
+
+@spaces.GPU(duration=40, progress=gr.Progress(track_tqdm=True))
+def gen_model_inputs(tokenizer, prompt, device):
+    system = "You are a helpful language and speech assistant. You are able to understand the speech content that the user provides, and assist the user with a variety of tasks using natural language."
+    DEFAULT_AUDIO_PATCH_TOKEN = "<audio_patch>"
+    audio_placeholder = DEFAULT_AUDIO_PATCH_TOKEN * CONFIG.audio_token_len
+    audio_placeholder = "\n"+audio_placeholder
+    audio_placeholder_ids = tokenizer(audio_placeholder).input_ids
+
+    begin_of_text_id = tokenizer.get_vocab()["<|begin_of_text|>"]
+    start_header_id = tokenizer.get_vocab()["<|start_header_id|>"]
+    end_header_id = tokenizer.get_vocab()["<|end_header_id|>"]
+    eot_id = tokenizer.get_vocab()["<|eot_id|>"]
+    nl_tokens = tokenizer('\n').input_ids
+    _system = tokenizer('system').input_ids
+    _user = tokenizer('user').input_ids
+    _assistant = tokenizer('assistant').input_ids
+
+    input_ids = []
+    input_id = []
+
+    system = [begin_of_text_id] + [start_header_id] + _system + [end_header_id] + nl_tokens + tokenizer(system).input_ids + [eot_id]
+    input_id += system
+
+    user_input_id = [start_header_id] + _user + [end_header_id] + audio_placeholder_ids + tokenizer(prompt).input_ids + [eot_id]
+    assistant_input_id = [start_header_id] + _assistant + [end_header_id] + nl_tokens
+
+    input_id += user_input_id
+    input_id += assistant_input_id
+
+    input_ids.append(input_id)
+    input_ids = torch.tensor(input_ids, dtype=torch.int).to(device)
+    attention_mask=input_ids.ne(tokenizer.pad_token_id)
+
+    return dict(input_ids=input_ids, attention_mask=attention_mask)
+
+@spaces.GPU(duration=40, progress=gr.Progress(track_tqdm=True))
+def inference(model, audio_processor, tokenizer, prompt, audio_path, device):
+    # apply chat template
+    model_inputs = gen_model_inputs(tokenizer, prompt, device)
+    model.cuda()
+    # audio preprocess
+    audio, _ = librosa.load(audio_path, sr=CONFIG.sampling_rate, mono=True)
+    audio_feat = audio_processor(
+        audio, sampling_rate=CONFIG.sampling_rate, return_tensors="pt"
+    ).input_features.to(device, dtype=torch.float16)
+    print(audio_feat)
+    output_ids = model.generate(
+        **model_inputs,
+        audios=audio_feat,
+        max_new_tokens=512,
+        eos_token_id=tokenizer.eos_token_id,
+        do_sample=True,
+        top_p=0.9,
+        temperature=0.2,
+    )
+
+    input_ids = model_inputs["input_ids"]
+    input_token_len = input_ids.shape[1]
+    n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+    if n_diff_input_output > 0:
+        print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+    outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+
+    outputs = outputs.strip()
+    if outputs.endswith(CONFIG.stop):
+        outputs = outputs[:-len(CONFIG.stop)]
+    outputs = outputs.strip()
+    
+    return outputs
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--adapter_size', type=int, default=1280)
+    parser.add_argument('--model_path', type=str, default="FreedomIntelligence/Soundwave")
+    args = parser.parse_args()
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    model_path = args.model_path
+    
+    model, audio_processor, tokenizer = load_model(model_path, device)
+
+    prompt = "Please transcribe the following audio and then answer based on the audio's transcription."
+    audio_path = "/mnt/nvme3n1/liuzhiheng/speech_copy/lzh/show_code/assets/audio/example_1.wav"
+
+    response = inference(model, audio_processor, tokenizer, prompt, audio_path, device)
+
+    print(f"{response}")

app.py

@@ -1,7 +1,66 @@
 import gradio as gr
+from Soundwave.run_inference import *
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+device = 'cuda'
+
+model, audio_processor, tokenizer = load_model("FreedomIntelligence/Soundwave", device)
+model.cuda()
+
+@spaces.GPU(duration=40, progress=gr.Progress(track_tqdm=True))
+def process_audio_text(text, audio):
+    # 音频路径是传入的文件路径
+    audio_path = audio  
+    print(audio_path)
+    system = "You are a helpful language and speech assistant. You are able to understand the speech content that the user provides, and assist the user with a variety of tasks using natural language."
+    if text == "" or text == " ":
+        text = "Please transcribe the following audio and then answer based on the audio's transcription."
+    response = inference(model, audio_processor, tokenizer, text, audio_path, device)
+    result = f"{response}"
+    return result
+
+examples = [  
+    ["Can you turn my English into German?", "./show_case/common_voice_en_19664034.mp3"],  # En-De
+    ["Can you identify the initial word that connects to 'currency_name' in this audio clip?", "./show_case/audio-1434542201-headset.wav"],  # ER
+    ["What do you think the speaker's message is intended to be in this audio?", "./show_case/audio-1434542201-headset.wav"],  # IC
+    ["What does the person say?", "./show_case/p225_002.wav"],  # DFake
+    # ["Assess whether this speech's pronunciation is Real or Fake.", "./show_case/Real.wav"],  # DFake
+    ["Assess whether this speech's pronunciation is Real or Fake.", "./show_case/Fake.wav"],  # DFake
+    ["What emotional weight does the speaker's tone carry?\nPick one answer from A, B, C, and D.\nA: fear\nB: sadness\nC: joy\nD: neutral", "./show_case/SER(emotion)_example.wav"],  #SER(emotion)
+    # ["Assess whether this speech's pronunciation is Real or Fake.", "./show_case/SVD_14154_file31512.mp3.wav_16k.wav_norm.wav_mono.wav_silence.wav"],  # SVD  
+    ["Choose the most suitable answer from options A, B, C, and D to respond the question in next line, you may only choose A or B or C or D.\nThe number of speakers delivering this speech is what?\nA. 4\nB. 2\nC.1\nD. 3", "./show_case/SNV_example.wav"],  #SNV
+    ["Identify the language of the conversation you just heard.","./show_case/SLR_example.wav"], #SLR
+    ["tell the gender of the speaker in this audio.","./show_case/SGR_018.wav"], #SGR
+    ["What's the sound we're hearing in this audio from?","./show_case/Sound_Vocal_example.wav"], #Sound_vocal
+    ["What is your best guess at the setting of this sound clip?","./show_case/Scene_example.wav"], #Sound_cochl
+    ["Choose the most suitable answer from options A, B, C, and D to respond the question in next line, Please think step by step and you may only choose A or B or C or D.\nRecognize the segment where 'project' is spoken by the speaker.\nA. [5.28, 5.39]\nB. [0.92, 1.39]\nC. [4.75, 5.28]\nD. [3.86, 4.23]","./show_case/SG_audio_1.wav"], #SG
+    ["What type of business does the first person's son have?","./show_case/SFT_Fisher_example.wav"] #SFT_Fisher
+]  
+
+with gr.Blocks() as demo:
+    gr.Markdown("""
+        <h1 style='text-align: center; color: #014377;'>🔊 Soundwave Demo</h1>
+        <p style='text-align: center;'>Upload an audio file and provide an instruction for the AI to process.</p>
+    """)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            audio_input = gr.Audio(label="🎤 Upload Audio", type="filepath", value="./show_case/p225_002.wav")
+        with gr.Column(scale=1):
+            text_input = gr.Textbox(label="📝 Enter text instruction", value="What does the person say?", lines=2)
+    
+    with gr.Row():
+        submit_button = gr.Button("🚀 Process Audio", size="lg")
+    
+    with gr.Row():
+        output_text = gr.Textbox(label="📜 Model output", lines=5, interactive=False)
+    
+    def handle_submit(text, audio):
+        return process_audio_text(text, audio)
+    
+    submit_button.click(fn=handle_submit, inputs=[text_input, audio_input], outputs=output_text)
+    
+    gr.Examples(examples, inputs=[text_input, audio_input])
+
+if __name__ == "__main__":
+    demo.launch()

show_case/Real.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d41d182a1a02e1402b4dba23978e769713982dcf39272aa0fd7451d21e312576
+size 214622

show_case/SER(emotion)_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fd43b0094b18a3d40230db994f78207877560c6b9b0ba2be694ede48712363
+size 884838

show_case/SFT_Fisher_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7afbc99ba77cb7d277154371c6a01a54a32a51b61097d5c0b7035061e4cf99ec
+size 583486

show_case/SGR_018.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fb25b832535382dd0bf82cfe337b3d1912ac0d0787b929ad1889b224246e16b
+size 1708140

show_case/SG_audio_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:427ca8e0dc94adf602a01a3513b44efba8136c0b5536e34b50f006bcd4d1d366
+size 197004

show_case/SLR_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb4b93187df340391151f6c9bf61401a866d1852d10ded839ce596a84a56a7a
+size 204204

show_case/SNV_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711a44be98d1fed2b38e9b6d2b4c8993a552f3c1f2cec9776f949bbb6aa8de9b
+size 259888

show_case/SVD_14154_file31512.mp3.wav_16k.wav_norm.wav_mono.wav_silence.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e14fc85c11123eee823a4182113331fd307e41012ca441825b0a6165c6b2105d
+size 108390

show_case/Scene_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee9f053c78f21958507e073cf7e7f184d37897407125dc69a71a940c6f99587
+size 882044

show_case/Sound_Vocal_example.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b83a3db215922aaa6a3ee9adb30fb04f0261b1956e16c4c1a3888809009a2b31
+size 112002

show_case/audio-1434542201-headset.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a946bbc783ab9411c52e9f2680919559fe31999ac9d3d65e24edadfd3a6d06
+size 102444

show_case/p225_002.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6716057eb1872e793eb5efe60f077cf19533fea634eea835f71a9ef180b0c2e2
+size 378156