Spaces:
Paused
Paused
File size: 7,146 Bytes
d708c77 8396ca1 359fce3 d708c77 6687492 d708c77 6687492 d708c77 6687492 d708c77 ba2ad0b 84316c3 d708c77 8396ca1 b0c18ca d708c77 b0c18ca d708c77 b0c18ca 8396ca1 d708c77 c5fe591 d708c77 7edde5e d708c77 c5fe591 d708c77 c5fe591 d224587 c5fe591 d224587 c5fe591 c62d4cd c5fe591 c62d4cd d708c77 c5fe591 d708c77 b0c18ca d708c77 c5fe591 b0c18ca d708c77 3315c31 3ddca39 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
# Copyright (2023) Tsinghua University, Bytedance Ltd. and/or its affiliates
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import gradio as gr
import spaces
import argparse
from model import SALMONN
class ff:
def generate(self, wav_path, prompt, prompt_pattern, num_beams, temperature, top_p):
print(f'wav_path: {wav_path}, prompt: {prompt}, temperature: {temperature}, num_beams: {num_beams}, top_p: {top_p}')
return "I'm sorry, but I cannot answer that question as it is not clear what you are asking. Can you please provide more context or clarify your question?"
parser = argparse.ArgumentParser()
parser.add_argument("--device", type=str, default="cuda:0")
parser.add_argument("--ckpt_path", type=str, default="./salmonn_7b_v0.pth")
parser.add_argument("--whisper_path", type=str, default="./whisper_large_v2")
parser.add_argument("--beats_path", type=str, default="./beats/BEATs_iter3_plus_AS2M_finetuned_on_AS2M_cpt2.pt")
parser.add_argument("--vicuna_path", type=str, default="./vicuna-7b-v1.5")
parser.add_argument("--low_resource", action='store_true', default=False)
parser.add_argument("--port", default=9527)
args = parser.parse_args()
args.low_resource = True # for huggingface A10 7b demo
# model = ff()
model = SALMONN(
ckpt=args.ckpt_path,
whisper_path=args.whisper_path,
beats_path=args.beats_path,
vicuna_path=args.vicuna_path,
low_resource=args.low_resource,
lora_alpha=28,
)
model.to(args.device)
model.eval()
@spaces.GPU(enable_queue=True)
def gradio_answer(speech, text_input, num_beams, temperature, top_p):
llm_message = model.generate(
wav_path=speech,
prompt=text_input,
num_beams=num_beams,
temperature=temperature,
top_p=top_p,
)
return llm_message[0]
title = """<h1 style="text-align: center;">SALMONN: Speech Audio Language Music Open Neural Network</h1>"""
image_src = """<h1 align="center"><a href="https://github.com/bytedance/SALMONN"><img src="https://raw.githubusercontent.com/bytedance/SALMONN/main/resource/salmon.png", alt="SALMONN" border="0" style="margin: 0 auto; height: 200px;" /></a> </h1>"""
description = """<h3 style="text-align: center;">This is a simplified gradio demo for <a href="https://huggingface.co/tsinghua-ee/SALMONN-7B" target="_blank">SALMONN-7B</a>. <br />To experience SALMONN-13B, you can go to <a href="https://bytedance.github.io/SALMONN">https://bytedance.github.io/SALMONN</a>.<br /> Upload your audio and ask a question!</h3>"""
css = """
div#col-container {
margin: 0 auto;
max-width: 840px;
}
"""
with gr.Blocks(css=css) as demo:
with gr.Column(elem_id="col-container"):
gr.HTML(title)
#gr.Markdown(image_src)
gr.HTML(description)
with gr.Row():
with gr.Column():
speech = gr.Audio(label="Audio", type='filepath')
with gr.Row():
text_input = gr.Textbox(label='User question', placeholder='Please upload your audio first', interactive=True)
submit_btn = gr.Button("Submit")
answer = gr.Textbox(label="Salmonn answer")
with gr.Accordion("Advanced Settings", open=False):
num_beams = gr.Slider(
minimum=1,
maximum=10,
value=4,
step=1,
interactive=True,
label="beam search numbers",
)
top_p = gr.Slider(
minimum=0.1,
maximum=1.0,
value=0.9,
step=0.1,
interactive=True,
label="top p",
)
temperature = gr.Slider(
minimum=0.8,
maximum=2.0,
value=1.0,
step=0.1,
interactive=False,
label="temperature",
)
with gr.Row():
examples = gr.Examples(
examples = [
["resource/audio_demo/gunshots.wav", "Recognize the speech and give me the transcription."],
["resource/audio_demo/gunshots.wav", "Listen to the speech and translate it into German."],
["resource/audio_demo/gunshots.wav", "Provide the phonetic transcription for the speech."],
["resource/audio_demo/gunshots.wav", "Please describe the audio."],
["resource/audio_demo/gunshots.wav", "Recognize what the speaker says and describe the background audio at the same time."],
["resource/audio_demo/gunshots.wav", "Use your strong reasoning skills to answer the speaker's question in detail based on the background sound."],
["resource/audio_demo/duck.wav", "Please list each event in the audio in order."],
["resource/audio_demo/duck.wav", "Based on the audio, write a story in detail. Your story should be highly related to the audio."],
["resource/audio_demo/duck.wav", "How many speakers did you hear in this audio? Who are they?"],
["resource/audio_demo/excitement.wav", "Describe the emotion of the speaker."],
["resource/audio_demo/mountain.wav", "Please answer the question in detail."],
["resource/audio_demo/jobs.wav", "Give me only three keywords of the text. Explain your reason."],
["resource/audio_demo/2_30.wav", "What is the time mentioned in the speech?"],
["resource/audio_demo/music.wav", "Please describe the music in detail."],
["resource/audio_demo/music.wav", "What is the emotion of the music? Explain the reason in detail."],
["resource/audio_demo/music.wav", "Can you write some lyrics of the song?"],
["resource/audio_demo/music.wav", "Give me a title of the music based on its rhythm and emotion."]
],
inputs=[speech, text_input]
)
text_input.submit(
gradio_answer, [speech, text_input, num_beams, temperature, top_p], [answer]
)
submit_btn.click(
gradio_answer, [speech, text_input, num_beams, temperature, top_p], [answer]
)
# demo.launch(share=True, enable_queue=True, server_port=int(args.port))
demo.queue(max_size=20).launch(share=False) |