Spaces:
Running
Running
Updated layout and added code comments
Browse files
app.py
CHANGED
@@ -1,58 +1,136 @@
|
|
|
|
1 |
from transformers import MusicgenForConditionalGeneration
|
2 |
from transformers import AutoProcessor
|
3 |
import scipy
|
4 |
import streamlit as st
|
5 |
|
6 |
-
#
|
7 |
import torch
|
8 |
torch.manual_seed(2912)
|
9 |
|
|
|
10 |
st.set_page_config(
|
11 |
-
page_title="Plant Orchestra
|
12 |
page_icon="🎵"
|
13 |
)
|
14 |
|
15 |
-
#
|
16 |
@st.cache_resource
|
17 |
def initialise_model():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
try:
|
|
|
19 |
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
20 |
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
21 |
return processor, model
|
22 |
except Exception as e:
|
|
|
23 |
st.error(f"Error initializing the model: {str(e)}")
|
24 |
return None, None
|
25 |
-
|
|
|
26 |
processor, model = initialise_model()
|
27 |
|
28 |
-
|
29 |
# Generate audio with given prompt
|
30 |
def generate_audio(processor, model, prompt):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
31 |
if processor is not None and model is not None:
|
32 |
try:
|
|
|
33 |
inputs = processor(
|
34 |
text=[prompt],
|
35 |
padding=True,
|
36 |
return_tensors="pt",
|
37 |
)
|
38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
39 |
return audio_values
|
40 |
except Exception as e:
|
|
|
41 |
st.error(f"Error generating audio: {str(e)}")
|
42 |
return None
|
43 |
|
44 |
-
#
|
45 |
def save_file(model, audio_values, filename):
|
46 |
-
|
47 |
-
|
48 |
|
|
|
|
|
|
|
|
|
49 |
|
|
|
|
|
50 |
|
51 |
-
|
52 |
-
|
|
|
|
|
|
|
|
|
|
|
53 |
|
54 |
|
55 |
-
|
|
|
|
|
|
|
56 |
if st.button("Generate Music"):
|
57 |
if processor is not None and model is not None:
|
58 |
with st.spinner("Generating audio..."):
|
@@ -87,8 +165,6 @@ st.sidebar.audio('sound/oak_leaf_fig.wav')
|
|
87 |
|
88 |
|
89 |
# Footer
|
90 |
-
st.
|
91 |
-
st.write()
|
92 |
-
st.write()
|
93 |
st.markdown("---")
|
94 |
st.markdown("Created with ❤️ by HS2912 W4 Group 2")
|
|
|
1 |
+
# Import necessary libraries
|
2 |
from transformers import MusicgenForConditionalGeneration
|
3 |
from transformers import AutoProcessor
|
4 |
import scipy
|
5 |
import streamlit as st
|
6 |
|
7 |
+
# Set a random seed for reproducibility
|
8 |
import torch
|
9 |
torch.manual_seed(2912)
|
10 |
|
11 |
+
# Configure the Streamlit app with a custom title and icon
|
12 |
st.set_page_config(
|
13 |
+
page_title="Plant Orchestra",
|
14 |
page_icon="🎵"
|
15 |
)
|
16 |
|
17 |
+
# Initialize the model for generating music
|
18 |
@st.cache_resource
|
19 |
def initialise_model():
|
20 |
+
"""
|
21 |
+
Initialize the model for generating music using Hugging Face Transformers.
|
22 |
+
|
23 |
+
This function loads a pre-trained processor and model from the "facebook/musicgen-small"
|
24 |
+
checkpoint. It is wrapped in a Streamlit cache, ensuring efficient resource management
|
25 |
+
for repeated usage of the model within the application.
|
26 |
+
|
27 |
+
Returns:
|
28 |
+
Tuple[Optional[AutoProcessor], Optional[MusicgenForConditionalGeneration]]: A tuple containing
|
29 |
+
the processor and model if initialization is successful. If an error occurs during
|
30 |
+
initialization, the function returns None for both elements of the tuple.
|
31 |
+
|
32 |
+
Example:
|
33 |
+
processor, model = initialise_model()
|
34 |
+
if processor is not None and model is not None:
|
35 |
+
# Model is successfully initialized, and you can use it for music generation.
|
36 |
+
pass
|
37 |
+
else:
|
38 |
+
# Handle initialization error.
|
39 |
+
pass
|
40 |
+
"""
|
41 |
try:
|
42 |
+
# Load the processor and model from the pretrained model checkpoint
|
43 |
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
44 |
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
45 |
return processor, model
|
46 |
except Exception as e:
|
47 |
+
# Handle any errors that may occur during model initialization
|
48 |
st.error(f"Error initializing the model: {str(e)}")
|
49 |
return None, None
|
50 |
+
|
51 |
+
# Call the 'initialise_model' function to set up the processor and model
|
52 |
processor, model = initialise_model()
|
53 |
|
|
|
54 |
# Generate audio with given prompt
|
55 |
def generate_audio(processor, model, prompt):
|
56 |
+
"""
|
57 |
+
Generate audio based on a given prompt using a pre-trained model.
|
58 |
+
|
59 |
+
This function takes a processor and model, which are responsible for processing
|
60 |
+
the input and generating audio, and a user-provided prompt to create a musical
|
61 |
+
composition.
|
62 |
+
|
63 |
+
Args:
|
64 |
+
processor (AutoProcessor): A pre-trained processor for text-to-sequence tasks.
|
65 |
+
model (MusicgenForConditionalGeneration): A pre-trained model for generating music.
|
66 |
+
prompt (str): The user-provided text prompt to guide music generation.
|
67 |
+
|
68 |
+
Returns:
|
69 |
+
Union[None, torch.Tensor]: If the audio generation is successful, it returns a
|
70 |
+
tensor containing the generated audio. If an error occurs during generation,
|
71 |
+
the function returns None.
|
72 |
+
|
73 |
+
Example:
|
74 |
+
processor, model = initialise_model()
|
75 |
+
if processor is not None and model is not None:
|
76 |
+
audio_data = generate_audio(processor, model, "Sunflower, temperature: 32.5 C, UV light intensity: 50%, Soil water level: 3cm/h")
|
77 |
+
if audio_data is not None:
|
78 |
+
# Use the generated audio for further processing or display.
|
79 |
+
pass
|
80 |
+
else:
|
81 |
+
# Handle audio generation error.
|
82 |
+
pass
|
83 |
+
else:
|
84 |
+
# Handle model initialization error.
|
85 |
+
pass
|
86 |
+
"""
|
87 |
if processor is not None and model is not None:
|
88 |
try:
|
89 |
+
# Prepare the input for the model by tokenizing and converting the text to tensors.
|
90 |
inputs = processor(
|
91 |
text=[prompt],
|
92 |
padding=True,
|
93 |
return_tensors="pt",
|
94 |
)
|
95 |
+
# Generate audio based on the processed input using the pre-trained model.
|
96 |
+
audio_values = model.generate(
|
97 |
+
**inputs.to("cpu"), # Ensure computation on the CPU
|
98 |
+
do_sample=True, # Enable sampling for creative output
|
99 |
+
guidance_scale=3, # Adjust the guidance scale (you can customize)
|
100 |
+
max_new_tokens=256, # Limit the length of generated audio (you can customize)
|
101 |
+
)
|
102 |
return audio_values
|
103 |
except Exception as e:
|
104 |
+
# Handle any exceptions that may occur during audio generation.
|
105 |
st.error(f"Error generating audio: {str(e)}")
|
106 |
return None
|
107 |
|
108 |
+
# Save audio file with scipy
|
109 |
def save_file(model, audio_values, filename):
|
110 |
+
"""
|
111 |
+
Save audio data as a WAV file using the SciPy library.
|
112 |
|
113 |
+
Args:
|
114 |
+
model: The pre-trained model used for audio generation.
|
115 |
+
audio_values (torch.Tensor): The tensor containing the generated audio data.
|
116 |
+
filename (str): The desired filename for the saved WAV file.
|
117 |
|
118 |
+
Returns:
|
119 |
+
None
|
120 |
|
121 |
+
Example:
|
122 |
+
save_file(model, audio_data, "generated_audio.wav")
|
123 |
+
"""
|
124 |
+
# Get the sampling rate from the model's configuration
|
125 |
+
sampling_rate = model.config.audio_encoder.sampling_rate
|
126 |
+
# Write the audio data to a WAV file with the specified filename
|
127 |
+
scipy.io.wavfile.write(filename, rate=sampling_rate, data=audio_values[0, 0].cpu().numpy())
|
128 |
|
129 |
|
130 |
+
# Main Code
|
131 |
+
st.title("Plant Orchestra 🌿")
|
132 |
+
st.markdown("Generate music based on your own terrarium plants.")
|
133 |
+
prompt = st.text_input(label='User input:', value='baby tears')
|
134 |
if st.button("Generate Music"):
|
135 |
if processor is not None and model is not None:
|
136 |
with st.spinner("Generating audio..."):
|
|
|
165 |
|
166 |
|
167 |
# Footer
|
168 |
+
st.markdown('##')
|
|
|
|
|
169 |
st.markdown("---")
|
170 |
st.markdown("Created with ❤️ by HS2912 W4 Group 2")
|