init app

README.md

@@ -2,11 +2,14 @@
 title: MMM MagicMusicMachine
 emoji: 🐨
 colorFrom: purple
-colorTo: indigo
+colorTo: magenta
 sdk: gradio
 sdk_version: 4.21.0
+python_version: python 3.9
+suggested_hardware: "a10g-large"
 app_file: app.py
 pinned: false
+tags: MusicAI, MultiModal, Audio, Text, Image
 license: mit
 ---
 

app.py

@@ -0,0 +1,205 @@
+import os
+
+import gradio as gr
+from gradio_components.prediction import load_model, predict
+
+theme = gr.themes.Glass(
+	primary_hue="fuchsia",
+	secondary_hue="indigo",
+	neutral_hue="slate",
+	font=[gr.themes.GoogleFont('Source Sans Pro'), 'ui-sans-serif', 'system-ui',
+	      'sans-serif'],
+	).set(
+	body_background_fill_dark='*background_fill_primary',
+	embed_radius='*table_radius',
+	background_fill_primary='*neutral_50',
+	background_fill_primary_dark='*neutral_950',
+	background_fill_secondary_dark='*neutral_900',
+	border_color_accent='*neutral_600',
+	border_color_accent_subdued='*color_accent',
+	border_color_primary_dark='*neutral_700',
+	block_background_fill='*background_fill_primary',
+	block_background_fill_dark='*neutral_800',
+	block_border_width='1px',
+	block_label_background_fill='*background_fill_primary',
+	block_label_background_fill_dark='*background_fill_secondary',
+	block_label_text_color='*neutral_500',
+	block_label_text_size='*text_sm',
+	block_label_text_weight='400',
+	block_shadow='none',
+	block_shadow_dark='none',
+	block_title_text_color='*neutral_500',
+	block_title_text_weight='400',
+	panel_border_width='0',
+	panel_border_width_dark='0',
+	checkbox_background_color_dark='*neutral_800',
+	checkbox_border_width='*input_border_width',
+	checkbox_label_border_width='*input_border_width',
+	input_background_fill='*neutral_100',
+	input_background_fill_dark='*neutral_700',
+	input_border_color_focus_dark='*neutral_700',
+	input_border_width='0px',
+	input_border_width_dark='0px',
+	slider_color='#2563eb',
+	slider_color_dark='#2563eb',
+	table_even_background_fill_dark='*neutral_950',
+	table_odd_background_fill_dark='*neutral_900',
+	button_border_width='*input_border_width',
+	button_shadow_active='none',
+	button_primary_background_fill='*primary_200',
+	button_primary_background_fill_dark='*primary_700',
+	button_primary_background_fill_hover='*button_primary_background_fill',
+	button_primary_background_fill_hover_dark='*button_primary_background_fill',
+	button_secondary_background_fill='*neutral_200',
+	button_secondary_background_fill_dark='*neutral_600',
+	button_secondary_background_fill_hover='*button_secondary_background_fill',
+	button_secondary_background_fill_hover_dark='*button_secondary_background_fill',
+	button_cancel_background_fill='*button_secondary_background_fill',
+	button_cancel_background_fill_dark='*button_secondary_background_fill',
+	button_cancel_background_fill_hover='*button_cancel_background_fill',
+	button_cancel_background_fill_hover_dark='*button_cancel_background_fill'
+	)
+
+
+_AUDIOCRAFT_MODELS = ["facebook/musicgen-melody",
+                      "facebook/musicgen-medium",
+                      "facebook/musicgen-small",
+                      "facebook/musicgen-large",
+                      "facebook/musicgen-melody-large"]
+
+
+def generate_prompt(difficulty, style):
+	_DIFFICULTY_MAPPIN = {
+		"Easy": "beginner player",
+		"Medum": "player who has 2-3 years experience",
+		"Hard": "player who has more than 4 years experiences"
+		}
+	prompt = 'piano only music for a {} to pratice with the touch of {}'.format(
+		_DIFFICULTY_MAPPIN[difficulty], style
+		)
+	return prompt
+
+def UI():
+	with gr.Blocks() as demo:
+		with gr.Tab("Generate Music by melody"):
+			with gr.Row():
+				with gr.Column():
+					with gr.Row():
+						model_path = gr.Dropdown(
+							choices=_AUDIOCRAFT_MODELS,
+							label="Select the model",
+							value="facebook/musicgen-melody-large"
+							)
+					with gr.Row():
+						duration = gr.Slider(
+							minimum=10,
+							maximum=60,
+							value=10,
+							label="Duration",
+							interactive=True
+							)
+					with gr.Row():
+						topk = gr.Number(label="Top-k", value=250, interactive=True)
+						topp = gr.Number(label="Top-p", value=0, interactive=True)
+						temperature = gr.Number(
+							label="Temperature", value=1.0, interactive=True
+							)
+						sample_rate = gr.Number(label="output music sample rate", value=32000, interactive=True)
+						difficulty = gr.Radio(["Easy", "Medium", "Hard"], label="Difficulty", value="Easy", interactive=True)
+						style = gr.Radio(["Jazz", "Classical Music", "Hip Hop", "Others"], value="Classical Music", label="music genre", interactive=True)
+						if style == "Others":
+							style = gr.Textbox(label="Type your music genre")
+						prompt = generate_prompt(difficulty.value, style.value)
+						customize = gr.Checkbox(
+							label="Customize the prompt", interactive=True
+							)
+						if customize:
+							prompt = gr.Textbox(label="Type your prompt")
+				with gr.Column():
+					with gr.Row():
+						melody = gr.Audio(
+							sources=["microphone", "upload"],
+							streaming=True,
+							label="Record or upload your audio",
+							interactive=True,
+							type="numpy",
+							show_label=True,
+						)
+					with gr.Row():
+						submit = gr.Button("Generate Music")
+						output = gr.Audio("listen to the generated music")
+						submit.click(fn=predict, inputs=melody, outputs=output)
+
+			generate_music = gr.Button("Generate Music")
+			output = gr.Audio("listen to the music")
+			generate_music.click(fn=predict,
+			                     inputs=[model_path, prompt, melody, duration, topk, topp, temperature, sample_rate],
+			                     outputs=output)
+
+		gr.Examples(
+			examples=[
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/twinkle_twinkle_little_stars_mozart.mp3"
+						),
+					"Easy",
+					32000,
+					20
+					],
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/golden_hour.mp3"
+						),
+					"Easy",
+					32000,
+					20
+					],
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/turkish_march_mozart.mp3"
+						),
+					"Easy",
+					32000,
+					20
+					],
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/golden_hour.mp3"
+						),
+					"Hard",
+					32000,
+					20
+					],
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/golden_hour.mp3"
+						),
+					"Hard",
+					32000,
+					40
+					],
+				[
+					os.path.join(
+						os.path.dirname(__file__),
+						"./data/audio/golden_hour.mp3"
+						),
+					"Hard",
+					16000,
+					20
+					],
+				],
+			inputs=[melody, difficulty, sample_rate, duration],
+			label="Audio Examples",
+			outputs=[output],
+			# cache_examples=True,
+			)
+	demo.queue().launch()
+
+if __name__ == "__main__":
+	UI()
+

data/audio/golden_hour.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90d863d50f73f0d84a2e199349909a2ef8e82cdec12c64e9ee29043e3f3a7730
+size 5468297

data/audio/turkish_march_mozart.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6367d283829147ff0db47b4bbf0fdd9d159ef95d40e5a9279c81e8be93f9f2cd
+size 5085237

data/audio/twinkle_twinkle_little_stars_mozart.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed46f25fb0031b270dafc14981e67121ecf094e15c6c6c138f7998672de8ce7a
+size 20276397

gradio_components/prediction.py

@@ -0,0 +1,103 @@
+import time
+import torch
+
+from audiocraft.data.audio_utils import convert_audio
+from audiocraft.data.audio import audio_write
+import gradio as gr
+from audiocraft.models import MusicGen
+
+from tempfile import NamedTemporaryFile
+from pathlib import Path
+
+
+def load_model(version='facebook/musicgen-melody'):
+    return MusicGen.get_pretrained(version)
+
+
+def _do_predictions(model, texts, melodies, duration, progress=False, gradio_progress=None, target_sr=32000, target_ac = 1, **gen_kwargs):
+    print("new batch", len(texts), texts, [None if m is None else (m[0], m[1].shape) for m in melodies])
+    be = time.time()
+    processed_melodies = []
+    for melody in melodies:
+        if melody is None:
+            processed_melodies.append(None)
+        else:
+            sr, melody = melody[0], torch.from_numpy(melody[1]).to(model.device).float().t()
+            if melody.dim() == 1:
+                melody = melody[None]
+            melody = melody[..., :int(sr * duration)]
+            melody = convert_audio(melody, sr, target_sr, target_ac)
+            processed_melodies.append(melody)
+
+    try:
+        if any(m is not None for m in processed_melodies):
+            # melody condition
+            outputs = model.generate_with_chroma(
+                descriptions=texts,
+                melody_wavs=processed_melodies,
+                melody_sample_rate=target_sr,
+                progress=progress,
+                return_tokens=False
+            )
+        else:
+            # text only
+            outputs = model.generate(texts, progress=progress, return_tokens=False)
+    except RuntimeError as e:
+        raise gr.Error("Error while generating " + e.args[0])
+    outputs = outputs.detach().cpu().float()
+    pending_videos = []
+    out_wavs = []
+    for output in outputs:
+        with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file:
+            audio_write(
+                file.name, output, model.sample_rate, strategy="loudness",
+                loudness_headroom_db=16, loudness_compressor=True, add_suffix=False)
+            out_wavs.append(file.name)
+    print("generation finished", len(texts), time.time() - be)
+    return out_wavs
+
+
+def predict(model_path, text, melody, duration, topk, topp, temperature, target_sr, progress=gr.Progress()):
+    global INTERRUPTING
+    global USE_DIFFUSION
+    INTERRUPTING = False
+    progress(0, desc="Loading model...")
+    model_path = model_path.strip()
+    if model_path:
+        if not Path(model_path).exists():
+            raise gr.Error(f"Model path {model_path} doesn't exist.")
+        if not Path(model_path).is_dir():
+            raise gr.Error(f"Model path {model_path} must be a folder containing "
+                           "state_dict.bin and compression_state_dict_.bin.")
+    if temperature < 0:
+        raise gr.Error("Temperature must be >= 0.")
+    if topk < 0:
+        raise gr.Error("Topk must be non-negative.")
+    if topp < 0:
+        raise gr.Error("Topp must be non-negative.")
+
+    topk = int(topk)
+    model = load_model(model_path)
+
+    max_generated = 0
+
+    def _progress(generated, to_generate):
+        nonlocal max_generated
+        max_generated = max(generated, max_generated)
+        progress((min(max_generated, to_generate), to_generate))
+        if INTERRUPTING:
+            raise gr.Error("Interrupted.")
+    model.set_custom_progress_callback(_progress)
+
+    wavs = _do_predictions(
+        [text],
+        [melody],
+        duration,
+        progress=True,
+        target_ac=1,
+        target_sr=target_sr,
+        top_k=topk,
+        top_p=topp,
+        temperature=temperature,
+        gradio_progress=progress)
+    return wavs

requirements.txt

@@ -0,0 +1,4 @@
+torch==2.1.0
+audiocraft
+basic-pitch
+gradio