push backend

back/.gitignore

@@ -0,0 +1,142 @@
+# Test files
+playing/
+test_waves/*
+data/*
+temp/*
+output/*
+
+comparison/
+reconstruction_waves/
+reconstruction_waves*
+
+
+# To preserve file for Dave :/
+librenderman.so
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# reference onlu
+models/docs/
+
+# large audio samples
+audio/large
+
+# final audio outputs
+audio/outputs
+
+# large datasets
+data/large
+
+# large saved models
+models/saved/large
+
+# mac
+.DS_Store
+
+# notebook experiments
+notebooks/experiments

back/.vscode/launch.json

@@ -0,0 +1,57 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "1 - Gen Config VST",
+      "type": "python",
+      "request": "launch",
+      "module": "generators.vst_generator",
+      "justMyCode": true,
+      "args": [
+        "generate"
+      ]
+    },
+    {
+      "name": "2 - Run VST",
+      "type": "python",
+      "request": "launch",
+      "module": "generators.vst_generator",
+      "justMyCode": true,
+      "args": [
+        "run",
+        "--config",
+        "plugin_config/TAL-NoiseMaker-config.json"
+      ]
+    },
+    {
+      "name": "3 - Train Debug",
+      "type": "python",
+      "request": "launch",
+      "module": "models.spectrogram_cnn",
+      "justMyCode": true,
+      "args": [
+        "--epoch",
+        "2000",
+        "--model",
+        "C6XL"
+      ]
+    },
+    {
+      "name": "4 - Debug Launch",
+      "type": "python",
+      "request": "launch",
+      "module": "models.spectrogram_cnn",
+      "justMyCode": true,
+      "args": [
+        "--epoch",
+        "1",
+        "--model",
+        "C6XL",
+        "--resume"
+      ]
+    }
+  ]
+}

back/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.analysis.autoImportCompletions": true,
+    "python.analysis.typeCheckingMode": "off",
+    "python.analysis.fixAll": ["source.unusedImports", "source.convertImportFormat"],
+    "editor.defaultFormatter": "ms-python.black-formatter"
+}

back/README.md

@@ -0,0 +1,38 @@
+# Inversynth Fork
+## AMP Team 
+
+## Launch instructions :
+
+*Optional: outputing your own config file for your VST* 
+```zsh
+python -m generators.vst_generator generate
+```
+*1. Dataset Creation based on config profile* 
+```zsh
+python -m generators.vst_generator run --config "your_config_path.json"
+```
+
+*2. Model training* 
+```zsh
+python -m generators.spectrogram_cnn --epoch "your_epoch_number" --model C6XL
+```
+
+Parameter | Default | Description
+---|---|---
+`--num_examples` | `2000` | Number of examples to create
+`--name` | `InverSynth` | Naming convention for datasets
+`--dataset_directory` | `test_datasets` | Directory for datasets
+`--wavefile_directory` | `test_waves` | Directory to for wave files.<br>Naming convention applied automatically
+`--length` | `1.0` | Length of each sample in seconds
+`--sample_rate` | `16384` | Sample rate (Samples/second)
+`--sampling_method` | `random` | Method to use for generating examples.<br>Currently only random, but may<br>include whole space later
+Optional |
+`--regenerate_samples` | | Regenerate the set of points to explore if it<br>exists (will also force regenerating audio)
+`--regenerate_audio` | | Regenerate audio files if they exist
+`--normalise` | | Apply audio normalization
+
+This module generates a dataset attempting to recreate the dataset generation<br>as defined in the [paper](paper/1812.06349.pdf)
+
+Selecting an architecture:
+
+- `C1`, `C2`, `C3`, `C4`, `C5`, `C6`, `C6XL`

back/generators/generator.py

@@ -0,0 +1,327 @@
+import argparse
+
+# ParamValue = Tuple[str,float,List[float]]
+import os
+import os.path
+from typing import List
+
+import h5py
+import numpy as np
+from scipy.io.wavfile import write as write_wav
+
+from generators.parameters import *
+
+"""
+This is a base class to derive different kinds of sound generator from (e.g.
+custom synthesis, VST plugins)
+"""
+
+
+class SoundGenerator:
+    """
+    This is now a wrapper round the 'real' generation function
+    to handle normalising and saving
+    """
+
+    def generate(
+        self,
+        parameters: dict,
+        filename: str,
+        length: float,
+        sample_rate: int,
+        extra: dict,
+        normalise: bool = True,
+    ) -> np.ndarray:
+        audio = self.do_generate(parameters, filename, length, sample_rate, extra)
+        if normalise:
+            max = np.max(np.absolute(audio))
+            if max > 0:
+                audio = audio / max
+        if not self.creates_wave_file():
+            self.write_file(audio, filename, sample_rate)
+
+    def do_generate(
+        self,
+        parameters: dict,
+        filename: str,
+        length: float,
+        sample_rate: int,
+        extra: dict,
+    ) -> np.ndarray:
+        print(
+            "Someone needs to write this method! Generating silence in {} with parameters:{}".format(
+                filename, str(parameters)
+            )
+        )
+        return np.zeros(int(length * sample_rate))
+
+    def creates_wave_file(self) -> bool:
+        return False
+
+    # Assumes that the data is -1..1 floating point
+    def write_file(self, data: np.ndarray, filename: str, sample_rate: int):
+        # REVIEW: is this needed?
+        # int_data = (data * np.iinfo(np.int16).max).astype(int)
+        write_wav(filename, sample_rate, data)
+
+
+"""
+This class runs through a parameter set, gets it to generate parameter settings
+then runs the sound generator over it.
+"""
+
+
+class DatasetCreator:
+    def __init__(
+        self,
+        name: str,
+        dataset_dir: str,
+        wave_file_dir: str,
+        parameters: ParameterSet,
+        normalise: bool = True,
+    ):
+        self.name = name
+        self.parameters = parameters
+        self.dataset_dir = dataset_dir
+        self.wave_file_dir = wave_file_dir
+        self.normalise = normalise
+        os.makedirs(dataset_dir, exist_ok=True)
+        os.makedirs(f"{wave_file_dir}/{name}", exist_ok=True)
+
+    def create_parameters(
+        self,
+        max: int = 2000,
+        method: str = "complete",
+        extra: dict = {},
+        force_create=False,
+    ) -> str:
+        filename = self.get_dataset_filename("data", "hdf5")
+        if os.path.isfile(filename) and not force_create:
+            print(
+                "Parameter file exists, not recreating (use --regenerate_samples if you want to force)"
+            )
+            return filename
+        print("+" * 40)
+        print(f"Generating Dataset {self.name}, {max} examples")
+        print(f"Datasets: {self.dataset_dir}")
+        print("+" * 40)
+
+        # Save out the parameters first
+        self.save_parameters()
+
+        # Generate the set of samples (could switch to generators,
+        # but need to figure out arbitrary size arrays in HDF5)
+        dataset: List[Sample] = []
+        # if method == "complete":
+            # dataset = self.parameters.recursively_generate_all()
+        # else:
+        dataset = self.parameters.sample_space(sample_size=max)
+
+        # Create the data file and add all the points to it
+        with h5py.File(filename, "w") as datafile:
+            # Figure out the sizes to store
+            records = len(dataset)
+            param_size = len(dataset[0].encode())
+
+            # Add columns to it
+            filenames = datafile.create_dataset(
+                "files", (records,), dtype=h5py.string_dtype()
+            )
+            parameters = datafile.create_dataset(
+                "parameters", (records,), dtype=h5py.string_dtype()
+            )
+            labels = datafile.create_dataset("labels", (records, param_size))
+            audio_exists = datafile.create_dataset(
+                "audio_exists", (records,), dtype=np.bool_
+            )
+
+            # Generate the sample points
+            for index, point in enumerate(dataset):
+                params = self.parameters.to_settings(point)
+                filenames[index] = self.get_wave_filename(index)
+                labels[index] = point.encode()
+                parameters[index] = json.dumps(params)
+                audio_exists[index] = False
+                if index % 1000 == 0:
+                    print("Generating parameters for example {}".format(index))
+            datafile.flush()
+        datafile.close()
+
+        return filename
+
+    def generate_audio(
+        self,
+        sound_generator: SoundGenerator,
+        length: float = 1,
+        sample_rate: int = 16384,
+        extra: dict = {},
+        dataset_filename=None,
+        force_generate=True,
+    ):
+        if dataset_filename is None:
+            dataset_filename = self.get_dataset_filename("data", "hdf5")
+
+        print("+" * 40)
+        print(
+            f"Generating Audio for Dataset {self.name} ({dataset_filename}), with {length}s at {sample_rate}/s"
+        )
+        print(f"Output waves: {self.wave_file_dir}, datasets: {self.dataset_dir}")
+        print("+" * 40)
+
+        with h5py.File(dataset_filename, "r+") as datafile:
+            for name, value in datafile.items():
+                print(f"{name}: {value}")
+            # Get the columns
+            filenames = datafile.get("files")
+            print(filenames)
+            parameters = datafile.get("parameters")
+            print(parameters)
+            audio_exists = datafile.get("audio_exists")
+            print(audio_exists)
+
+            for index, filename in enumerate(filenames):
+                if (
+                    audio_exists[index]
+                    and os.path.isfile(filename)
+                    and not force_generate
+                ):
+                    print(f"Audio exists for index {index} ({filename})")
+                else:
+                    print(f"Generating Audio for index {index} ({filename})")
+                    params = json.loads(parameters[index])
+                    audio = sound_generator.generate(
+                        params,
+                        filename,
+                        length,
+                        sample_rate,
+                        extra,
+                        normalise=self.normalise,
+                    )
+                    audio_exists[index] = bool(audio)
+                    datafile.flush()
+                if index % 1000 == 0:
+                    print("Generating example {}".format(index))
+
+    def save_parameters(self):
+        self.parameters.save_json(self.get_dataset_filename("params", "json"))
+        self.parameters.save(self.get_dataset_filename("params", "pckl"))
+
+    def get_dataset_filename(self, type: str, extension: str = "txt") -> str:
+        return f"{self.dataset_dir}/{self.name}_{type}.{extension}"
+
+    def get_wave_filename(self, index: int) -> str:
+        return f"{self.wave_file_dir}/{self.name}/{self.name}_{index:05d}.wav"
+
+
+def default_generator_argparse():
+    parser = argparse.ArgumentParser(description="Process some integers.")
+    parser.add_argument(
+        "--num_examples",
+        type=int,
+        dest="samples",
+        action="store",
+        default=20000,
+        help="Number of examples to create",
+    )
+    parser.add_argument(
+        "--name",
+        type=str,
+        dest="name",
+        default="InverSynth",
+        help="Name of datasets to create",
+    )
+    parser.add_argument(
+        "--dataset_directory",
+        type=str,
+        dest="data_dir",
+        default="test_datasets",
+        help="Directory to put datasets",
+    )
+    parser.add_argument(
+        "--wavefile_directory",
+        type=str,
+        dest="wave_dir",
+        default="test_waves",
+        help="Directory to put wave files. Will have the dataset name appended automatically",
+    )
+    parser.add_argument(
+        "--length",
+        type=float,
+        dest="length",
+        default=1.0,
+        help="Length of each sample in seconds",
+    )
+    parser.add_argument(
+        "--sample_rate",
+        type=int,
+        dest="sample_rate",
+        default=16384,
+        help="Sample rate (Samples/second)",
+    )
+    parser.add_argument(
+        "--sampling_method",
+        type=str,
+        dest="method",
+        default="random",
+        choices=["random"],
+        help="Method to use for generating examples. Currently only random, but may include whole space later",
+    )
+    parser.add_argument(
+        "--regenerate_samples",
+        action="store_true",
+        help="Regenerate the set of points to explore if it exists (will also force regenerating audio)",
+    )
+    parser.add_argument(
+        "--regenerate_audio",
+        action="store_true",
+        help="Regenerate audio files if they exists",
+    )
+    parser.add_argument(
+        "--normalise", action="store_true", help="Regenerate audio files if they exists"
+    )
+    return parser
+
+
+def generate_examples(
+    gen: SoundGenerator, parameters: ParameterSet, args=None, extra={}
+):
+    if not args:
+        parser = default_generator_argparse()
+        args = parser.parse_args()
+
+    g = DatasetCreator(
+        name=args.name,
+        dataset_dir=args.data_dir,
+        wave_file_dir=args.wave_dir,
+        parameters=parameters,
+        normalise=args.normalise,
+    )
+
+    g.create_parameters(
+        max=args.samples, method=args.method, force_create=True
+    )
+
+    g.generate_audio(
+        sound_generator=gen,
+        length=args.length,
+        sample_rate=args.sample_rate,
+        extra=extra,
+        force_generate=args.regenerate_audio | args.regenerate_samples,
+    )
+
+
+if __name__ == "__main__":
+    gen = SoundGenerator()
+    parameters = ParameterSet(
+        [
+            Parameter("p1", [100, 110, 120, 130, 140]),
+            Parameter("p2", [200, 220, 240, 260, 280]),
+        ]
+    )
+    g = DatasetCreator(
+        "example_generator",
+        dataset_dir="test_datasets",
+        wave_file_dir="test_waves/example/",
+        parameters=parameters,
+    )
+    g.generate_examples(sound_generator=gen, parameters=parameters)

back/generators/parameters.py

@@ -0,0 +1,176 @@
+import json
+import random
+from dataclasses import dataclass
+from pickle import dump
+from typing import Dict, List, Sequence, Tuple
+
+import numpy as np
+
+"""
+A setting for a parameter, with its oneHOT encoding
+"""
+
+
+@dataclass
+class ParamValue:
+    name: str
+    value: float
+    encoding: List[float]
+
+
+"""
+A sample point - the parameter values, the oneHOT encoding and the audio
+"""
+
+
+@dataclass
+class Sample:
+    # parameter_values: List[Tuple[str,float]]
+    # parameter_encoding:List[List[float]]
+    parameters: List[ParamValue]
+    # length:float=0.1
+    # sample_rate:int = 44100
+    # audio:np.ndarray = np.zeros(1)
+
+    def value_list(self) -> List[Tuple[str, float]]:
+        return [(p.name, p.value) for p in self.parameters]
+
+    def encode(self) -> List[float]:
+        return np.hstack([p.encoding for p in self.parameters])
+
+
+class Parameter:
+    def __init__(self, name: str, levels: list, id=""):
+        self.name = name
+        self.levels = levels
+        self.id = id
+
+    def get_levels(self) -> List[ParamValue]:
+        return [self.get_value(i) for i in range(len(self.levels))]
+
+    def sample(self) -> ParamValue:
+        index: int = random.choice(range(len(self.levels)))
+        return self.get_value(index)
+
+    def get_value(self, index: int) -> ParamValue:
+        encoding = np.zeros(len(self.levels)).astype(float)
+        encoding[index] = 1.0
+        return ParamValue(
+            name=self.name,
+            # Actual value
+            value=self.levels[index],
+            # One HOT encoding
+            encoding=encoding,
+        )
+
+    def decode(self, one_hot: List[float]) -> ParamValue:
+        ind = np.array(one_hot).argmax()
+        # ind = tf.cast(tf.argmax(one_hot, axis=-1), "int32")
+        return self.get_value(ind)
+
+    def from_output(
+        self, current_output: List[float]
+    ) -> Tuple[ParamValue, List[float]]:
+        param_data = current_output[: len(self.levels)]
+        remaining = current_output[len(self.levels) :]
+        my_val = self.decode(param_data)
+        return (my_val, remaining)
+
+    def to_json(self):
+        return {"name": self.name, "levels": self.levels, "id": self.id}
+
+
+class ParameterSet:
+    def __init__(self, parameters: List[Parameter], fixed_parameters: dict = {}):
+        self.parameters = parameters
+        self.fixed_parameters = fixed_parameters
+
+    def sample_space(self, sample_size=2000) -> Sequence[Sample]:
+        print("Sampling {} points from parameter space".format(sample_size))
+        dataset = []
+        for i in range(sample_size):
+            params = [p.sample() for p in self.parameters]
+            dataset.append(Sample(params))
+            if i % 1000 == 0:
+                print("Sampling iteration: {}".format(i))
+        return dataset
+
+    # Runs through the whole parameter space, setting up parameters and calling the generation function
+    # Excuse slightly hacky recusions - sure there's a more numpy-ish way to do it!
+    def recursively_generate_all(
+        self, parameter_list: list = None, parameter_set=[], return_list=[]
+    ) -> Sequence[Sample]:
+        print("Generating entire parameter space")
+        if parameter_list is None:
+            parameter_list = self.parameters
+        param = parameter_list[0]
+        remaining = parameter_list[1:]
+        for p in param.levels:
+            ps = parameter_set.copy()
+            ps.append((param.name, p))
+            if len(remaining) == 0:
+                return_list.append(ps)
+            else:
+                self.recursively_generate_all(remaining, ps, return_list)
+        return return_list
+
+    def to_settings(self, p: Sample):
+        params = self.fixed_parameters.copy()
+        params.update(dict(p.value_list()))
+        return params
+
+    def encoding_to_settings(self, output: List[float]) -> Dict[str, float]:
+        params = self.fixed_parameters.copy()
+        for p in self.decode(output):
+            params[p.name] = p.value
+        return params
+
+    def decode(self, output: List[float]) -> List[ParamValue]:
+        values = []
+        for p in self.parameters:
+            v, output = p.from_output(output)
+            values.append(v)
+        if len(output) > 0:
+            print("Leftover output!: {}".format(output))
+        return values
+
+    def save(self, filename):
+        with open(filename, "wb") as file:
+            dump(self, file)
+
+    def save_json(self, filename):
+        dump = self.to_json()
+        with open(filename, "w") as file:
+            json.dump(dump, file, indent=2)
+
+    def explain(self):
+        levels = 0
+        for p in self.parameters:
+            levels += len(p.levels)
+        return {
+            "n_variable": len(self.parameters),
+            "n_fixed": len(self.fixed_parameters),
+            "levels": levels,
+        }
+
+    def to_json(self):
+        return {
+            "parameters": [p.to_json() for p in self.parameters],
+            "fixed": self.fixed_parameters,
+        }
+
+
+"""
+Generates evenly spaced parameter values
+paper:
+The rest of the synthesizer parameters ranges are quantized evenly to 16
+classes according to the following ranges ...
+For each parameter, the first and last classes correspond to its range limits
+"""
+
+
+def param_range(steps, min, max):
+    ext = float(max - min)
+    return [n * ext / (steps - 1) + min for n in range(steps)]
+
+

back/generators/vst_generator.py

@@ -0,0 +1,235 @@
+import json
+import re
+
+import dawdreamer as rm
+import librosa
+import numpy as np
+import pandas as pd
+
+
+from generators.generator import *
+from generators.parameters import *
+
+
+class VSTGenerator(SoundGenerator):
+    def __init__(
+        self,
+        vst: str,
+        sample_rate,
+        randomise_non_set: bool = True,
+        randomise_all: bool = False,
+    ):
+        self.vst = vst
+        self.randomise_non_set = randomise_non_set
+        self.randomise_all = randomise_all
+        self.sample_rate = sample_rate
+        self.load_engine()
+
+    def load_engine(self):
+        print("_____ LOADING VST _______")
+        engine = rm.RenderEngine(self.sample_rate, 1024)
+        synth = engine.make_plugin_processor("my_synth", self.vst)
+        if synth:
+            print("Loaded {}".format(self.vst))
+
+            self.engine = engine
+            self.synth = synth
+        else:
+            print("Couldn't load VST {}".format(self.vst))
+        print("_____ LOADED VST _______")
+
+    # def do_sound_generation(self,parameter_set,base_filename)->np.ndarray:
+    def do_generate(
+        self,
+        parameters: dict,
+        filename: str,
+        length: float,
+        sample_rate: int,
+        extra: dict = {},
+    ) -> np.ndarray:
+        if not self.engine:
+            print("VST not loaded")
+            return np.zeros(5)
+        resample = False
+        if not self.sample_rate == sample_rate:
+            resample = True
+        synth = self.synth
+        engine = self.engine
+        # print( synth.get_parameters_description() )
+        # print("Params to set:{}".format(parameters))
+
+        ids = dict([(p["name"], p["id"]) for p in extra["config"]["fixed_parameters"]])
+        ids.update(dict([(p["name"], p["id"]) for p in extra["config"]["parameters"]]))
+
+        # if self.randomise_non_set:
+        # new_patch = self.patch_generator.get_random_patch()
+        # engine.set_patch(new_patch)
+
+        synth_params = dict(synth.get_patch())
+        # Start with defaults
+
+        # if not self.randomise_non_set:
+        # for i in range(84):
+        #     synth_params[i] = 0.5
+
+        for name, value in parameters.items():
+            synth_params[ids[name]] = value
+
+        # if self.randomise_all:
+        #     new_patch = self.patch_generator.get_random_patch()
+        #     engine.set_patch(new_patch)
+
+        note_length = length * 0.8
+        if "note_length" in extra:
+            note_length = extra["note_length"]
+
+        synth.set_patch(list(synth_params.items()))
+        synth.add_midi_note(40, 127, 0.1, note_length)
+        # don't do reverb
+        graph = [
+            (synth, []),  # synth takes no inputs, so we give an empty list.
+        ]
+
+        engine.load_graph(graph)
+        engine.render(1)
+        data = engine.get_audio()
+        df = pd.DataFrame(data)
+        try:
+            data = librosa.to_mono(data).transpose()
+        except:
+            print("ERROR" * 100)
+            df = df.fillna(0)
+            data = df.to_numpy()
+            data = librosa.to_mono(data).transpose()
+        
+        df = pd.DataFrame(data)
+        if(librosa.util.valid_audio(data)):
+            nsamps_target = int(1.0 * sample_rate)
+            # print(f"Got {len(data)} frames as type {type(data)}. Target: {nsamps_target}")
+            result = np.array(data)
+            return result
+        # else:
+        #     print("ERROR" * 100)
+        #     df = df.fillna(0)
+        #     data = df.to_numpy()
+        #     return data
+
+    
+
+    def create_config(self, filename='default_config.json', default_value=0.0):
+        r = re.compile("(.*): (.*)")
+        params = []
+        fixed = []
+        for line in self.synth.get_parameters_description():
+            line['defaultValue']=float(line['defaultValue'])
+            if line['index'] < 86:
+            #     fixed.append(
+            #     {"id": line['index'], "name": line['name'], "value": line['defaultValue']}
+            #     )
+            # else:    
+                params.append(
+                    {"id": line['index'], "name": line['name'], "value": line['defaultValue']}
+                )
+        output = {"parameters": params, "fixed_parameters": fixed}
+        os.makedirs("plugin_config", exist_ok=True)
+        with open("plugin_config/gen_config_"+str(self.vst)+'.json', "w") as f:
+            json.dump(output, f, indent=4)
+        return output
+
+
+# Run the generator to create a full dataset
+def run_generator(args):  # name: str, plugin: str, config: str, max: int,
+    # dataset_directory: str, wavefile_directory: str,
+    # sample_rate: int = 16384, length: float = 1.0, note_length: float = -1, method: str = 'random'):
+
+    note_length = args.note_length
+    if note_length < 0.0:
+        note_length = note_length * 0.8
+
+    with open(args.config_file, "r") as f:
+        config = json.load(f)
+        sample=[]
+        for p in config['parameters']:
+            if isinstance(p['values'], str): 
+                sample.append(
+                    Parameter(p["name"], param_range(16, 0, 1), p.get("id", ""))
+                )
+            elif isinstance(p['values'], list): 
+                sample.append(
+                    Parameter(p["name"], p['values'], p.get("id", ""))
+                )
+                        
+        fixed = dict([(p["name"], p["value"]) for p in config["fixed_parameters"]])
+
+        plugin_rate = args.generate_samplerate or args.sample_rate
+
+    generate_examples(
+        gen=VSTGenerator(vst=args.plugin, sample_rate=16384),
+        parameters=ParameterSet(parameters=sample, fixed_parameters=fixed),
+        args=args,
+        extra={"note_length": note_length, "config": config},
+    )
+
+
+# Create blank config file based on the plugin's parameter sets
+def generate_defaults(plugin: str, output: str, default: float = 0.5):
+    gen = VSTGenerator(vst=plugin, sample_rate=16384)
+    gen.create_config(output, default_value=default)
+
+
+# Example: python -m generators.vst_generator run --plugin /Library/Audio/Plug-Ins/VST/Lokomotiv.vst --config plugin_config/lokomotiv.json --dataset_name explore --wavefile_directory "test_waves/explore"
+
+if __name__ == "__main__":
+    pass
+    # parser = argparse.ArgumentParser(description='Process some integers.')
+    parser = default_generator_argparse()
+    parser.add_argument(
+        "command",
+        type=str,
+        choices=["run", "generate"],
+        help="action to take: run (run the generator with a config) or generate (generate a blank config file for the plugin)",
+    )
+    parser.add_argument(
+        "--plugin",
+        dest="plugin",
+        default='libTAL-NoiseMaker.so',
+        help='plugin file. .so on linux, on mac its the top level plugin dir, e.g. "/Library/Audio/Plug-Ins/VST/Lokomotiv.vst"',
+    )
+    parser.add_argument(
+        "--output", dest="outfile", help="Place to store the generated parameters file"
+    )
+    parser.add_argument("--config", dest="config_file", help="Config file to use")
+    parser.add_argument(
+        "--default_value",
+        type=float,
+        dest="default_param",
+        action="store",
+        default=0.5,
+        help="Default setting for parameters when generating a blank config",
+    )
+    parser.add_argument(
+        "--note_length",
+        type=float,
+        dest="note_length",
+        default=0.8,
+        help="Length of a note in seconds",
+    )
+    parser.add_argument(
+        "--generation_sample_rate",
+        type=int,
+        default=None,
+        dest="generate_samplerate",
+        help="Sample rate for audio generation. Defaults to target samplerate, but some plugins (Dexed) have trouble running a our funny sample rates. Will be resampled to the target rate after generation",
+    )
+
+    args = parser.parse_args()
+    print(args)
+    if args.command == "run":
+        run_generator(args)
+        # args.name, args.plugin, args.config_file,
+        #          args.samples, args.data_dir, args.wave_dir)
+
+    if args.command == "generate":
+        generate_defaults(args.plugin, args.outfile, args.default_param)
+    quit()
+   

back/main.py

@@ -0,0 +1,179 @@
+from contextlib import asynccontextmanager
+import uuid
+from fastapi import Depends, FastAPI, File, HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+import os
+from starlette.exceptions import HTTPException as StarletteHTTPException
+from fastapi.middleware.cors import CORSMiddleware
+import pathlib
+from contextlib import asynccontextmanager
+from glob import glob
+
+
+from models.launch import inferrence, train_model
+from models.spectrogram_cnn import get_model
+
+# distinguish model type for reshaping
+
+SERVER = "http://localhost:7860/"
+
+path = os.path.dirname(os.path.realpath(__file__))
+
+def load_model_and_parameters():
+    setup = {
+        "model_name": "C6XL",
+        "dataset_name": "InverSynth",
+        "epochs": 1,
+        "dataset_dir": "test_datasets",
+        "output_dir": "output",
+        "dataset_file": None,
+        "parameters_file": None,
+        "data_format": "channels_last",
+        "run_name": None,
+        "resume": True,
+    }
+    setup["model_type"] = "STFT"
+
+    try:
+        # charger model
+        model, parameters_file = train_model(model_callback=get_model, **setup)
+    except Exception as e:
+        print(f"Couldn't load model: {e}")
+        return None, None
+
+    return model, parameters_file
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+
+    
+
+    # Remove all files in the temp folder
+    tempFolderPath = os.path.join(path, "temp")
+    if os.path.exists(tempFolderPath):
+        for file_name in os.listdir(tempFolderPath):
+            file_path = os.path.join(tempFolderPath, file_name)
+            try:
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+            except Exception as e:
+                print(f"Error deleting file {file_path}: {e}")
+
+    if not os.path.exists(tempFolderPath):
+        os.makedirs(tempFolderPath)
+
+    yield
+ 
+app = FastAPI(lifespan=lifespan)
+
+str_p = str(path)
+
+
+class SPAStaticFiles(StaticFiles):
+    async def get_response(self, path: str, scope):
+        try:
+            return await super().get_response(path, scope)
+        except (HTTPException, StarletteHTTPException) as ex:
+            if ex.status_code == 404:
+                return await super().get_response("index.html", scope)
+            else:
+                raise ex
+
+
+
+
+
+@app.get("/download/{file_id}")
+async def generate_audio(file_id: str):
+    try:
+        # Use glob to find files starting with the specified ID
+        matching_files = glob(f"temp/{file_id}*")
+
+        if not matching_files:
+            # Handle the case when no matching file is found
+            print(f"No file found for file ID {file_id}")
+            raise HTTPException(status_code=404, detail="File not found")
+
+        # Assuming you want to copy the first matching file
+        else:
+            source_file_path = matching_files[0]
+            # Check if the file exists
+            # You can perform additional processing or send the file directly
+            return JSONResponse(content={"url": f"{source_file_path}"})
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+def is_valid_audio(file_extension):
+    # Define a list of valid audio file extensions
+    valid_audio_extensions = [".mp3", ".wav", ".ogg", ".flac"]
+
+    # Check if the provided file extension is in the list of valid audio extensions
+    return file_extension.lower() in valid_audio_extensions
+
+
+@app.post("/upload/")
+async def upload_audio_file(file: UploadFile = File(...)):
+    try:
+        model, parameters_file = load_model_and_parameters()
+    except:
+        raise("Couldn't load model")
+    try:
+        # Create a unique identifier for the uploaded file
+        file_id = str(uuid.uuid4())
+        
+        # Extract the original file extension
+        _, file_extension = os.path.splitext(file.filename)
+
+        # Check if the file has a valid audio extension
+        if not is_valid_audio(file_extension):
+            raise HTTPException(status_code=400, detail="Invalid audio file format")
+
+        
+        # Construct the file paths with the original file extension
+        file_path = os.path.join("temp", file_id + file_extension)
+
+        
+        with open(file_path, "wb") as audio_file:
+            audio_file.write(file.file.read())
+        # generate_output_audio(file_path, output_file_path)
+        output = await start_inference(model=model, parameters_file=parameters_file, file_id=file_id, file_extension=file_extension)
+        # Send a confirmation with the identifier
+        return {"file_path": SERVER+output[0], "csv_path": SERVER+output[1], "output_file_path": SERVER+output[2]}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+async def start_inference(model, parameters_file, file_id: str, file_extension : str):
+    file_path = os.path.join("temp", file_id + file_extension)
+    
+    output = inferrence(model=model, parameters_file=parameters_file, file_path=file_path, file_id=file_id)
+    
+    return output
+
+
+origins = ["*"]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=origins,
+    allow_headers=origins,
+)
+
+app.mount(
+    "/temp", StaticFiles(directory="temp", check_dir=True, html=True), name="temp"
+)
+app.mount(
+    "/",
+    SPAStaticFiles(directory=f"{pathlib.PurePath(str_p).parent}/front/dist", html=True),
+    name="dist",
+)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=7860)

back/models/app.py

@@ -0,0 +1,645 @@
+import datetime
+import json
+import logging
+import os
+from pickle import load
+from typing import Callable, List
+import librosa
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from tensorflow import keras
+from keras import backend as K
+from keras.callbacks import CSVLogger
+from kapre.time_frequency import Spectrogram
+from models.importer_audio import audio_importer
+import dawdreamer as daw
+from scipy.io import wavfile
+import librosa
+
+
+from generators.parameters import ParameterSet, ParamValue
+from models.common.data_generator import SoundDataGenerator
+
+
+weight_var = K.variable(0.)
+
+class Weight_trans(keras.callbacks.Callback):
+    def __init__(self, weight_var, transition, epochs):
+        self.alpha = weight_var
+        self.transition = transition
+        self.epochs = epochs  
+    def on_epoch_end(self, epoch, logs={}):
+        if epoch > 680:
+            if self.transition == "linear":
+                K.set_value(self.alpha, ((epoch)/(self.epochs) - 0.617)*0.00001)
+                tf.print(f"new weight {weight_var.numpy()}")
+            if self.transition == "linear2":
+                K.set_value(self.alpha, (1.5625*epoch - 1.0625)*0.00001)
+                tf.print(f"new weight {weight_var.numpy()}")
+            if self.transition == "log":
+                K.set_value(self.alpha, (1- (tf.math.log(epoch*0.001 - 0.67285)/tf.math.log(0.0005)) - 0.35)*0.00001)
+                tf.print("log")
+            if self.transition == "log2":
+                K.set_value(self.alpha, (1- (tf.math.log(epoch*0.001 - 0.6575)/tf.math.log(0.0005)) - 0.5)*0.00001)
+                tf.print("log")    
+            if self.transition == "log3":
+                K.set_value(self.alpha, (1- (tf.math.log(epoch*0.001 - 0.67978)/tf.math.log(0.00000005)) - 0.5)*0.00001)
+                tf.print("log")             
+            if self.transition == "square":
+                K.set_value(self.alpha, 4.1*tf.pow(epoch*0.001 - 0.65, 2) + 0.002)
+                print("exp")
+            if self.transition == "quad":
+                K.set_value(self.alpha, 33*tf.pow(epoch*0.001 - 0.65, 4) + 0.002)
+                print("quad")
+
+
+def train_val_split(
+    x_train: np.ndarray, y_train: np.ndarray, split: float = 0.2,
+) -> tuple:
+
+    slice: int = int(x_train.shape[0] * split)
+
+    x_val: np.ndarray = x_train[-slice:]
+    y_val: np.ndarray = y_train[-slice:]
+
+    x_train = x_train[:-slice]
+    y_train = y_train[:-slice]
+
+    return (x_val, y_val, x_train, y_train)
+
+
+"""Model Utils"""
+
+
+def mean_percentile_rank(y_true, y_pred, k=5):
+    """
+    @paper
+    The first evaluation measure is the Mean Percentile Rank
+    (MPR) which is computed per synthesizer parameter.
+    """
+    # TODO
+
+
+def top_k_mean_accuracy(y_true, y_pred, k=5):
+    """
+    @ paper
+    The top-k mean accuracy is obtained by computing the top-k
+    accuracy for each test example and then taking the mean across
+    all examples. In the same manner as done in the MPR analysis,
+    we compute the top-k mean accuracy per synthesizer
+    parameter for 𝑘 = 1, ... ,5.
+    """
+    # TODO: per parameter?
+    original_shape = tf.shape(y_true)
+    y_true = tf.reshape(y_true, (-1, tf.shape(y_true)[-1]))
+    y_pred = tf.reshape(y_pred, (-1, tf.shape(y_pred)[-1]))
+    top_k = K.in_top_k(y_pred, tf.cast(tf.argmax(y_true, axis=-1), "int32"), k)
+    correct_pred = tf.reshape(top_k, original_shape[:-1])
+    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))
+
+@tf.function
+def CustomLoss(y_true, y_pred):
+    bce = tf.keras.losses.BinaryCrossentropy()
+    weights = custom_spectral_loss(y_true, y_pred)
+    weight_shift = (1-weight_var.numpy())+(weight_var.numpy()*weights.numpy())
+    # tf.print(f"New weight is {weight_shift}")
+    loss = bce(y_true, y_pred, sample_weight=weight_shift)
+    return loss  
+
+@tf.function
+def custom_spectral_loss(y_true, y_pred):
+    # tf.print("After compiling model :",tf.executing_eagerly())
+
+    y_true = tf.reshape(y_true, (-1, tf.shape(y_true)[-1]))
+    y_pred = tf.reshape(y_pred, (-1, tf.shape(y_pred)[-1]))
+
+        
+    # Assuming y_true and y_pred contain parameters for audio synthesis
+    # Extract parameters from y_true and y_pred    
+    with open("test_datasets/InverSynth_params.pckl", "rb") as f:
+        parameters: ParameterSet = load(f)
+    
+    predlist_true: List[ParamValue] = parameters.decode(y_true[0])
+
+    predlist_pred: List[ParamValue] = parameters.decode(y_pred[0])
+    
+    # Convert parameter lists to DataFrames
+    # Generate audio from parameters
+    audio_true, penalty = generate_audio(predlist_true)
+    audio_pred, penalty = generate_audio(predlist_pred)
+
+    # Compute spectrogram
+    if SPECTRO_TYPE == 'spectro':
+        spectrogram_true = tf.math.abs(tf.signal.stft(audio_true, frame_length=1024, frame_step=512))
+        spectrogram_pred = tf.math.abs(tf.signal.stft(audio_pred, frame_length=1024, frame_step=512))
+    elif SPECTRO_TYPE == 'qtrans':
+        spectrogram_true = librosa.amplitude_to_db(librosa.cqt(audio_true, sr=SAMPLE_RATE, hop_length=128), ref=np.max)
+        spectrogram_pred = librosa.amplitude_to_db(librosa.cqt(audio_pred, sr=SAMPLE_RATE, hop_length=128), ref=np.max)
+    elif SPECTRO_TYPE == 'mel':
+        mel_spect = librosa.feature.melspectrogram(audio_true, sr=SAMPLE_RATE, n_fft=2048, hop_length=1024)
+        spectrogram_true = librosa.power_to_db(mel_spect, ref=np.max)
+        mel_spect = librosa.feature.melspectrogram(audio_pred, sr=SAMPLE_RATE, n_fft=2048, hop_length=1024)
+        spectrogram_pred = librosa.power_to_db(mel_spect, ref=np.max)
+    #L1 LOSS
+    if LOSS_TYPE == 'L1':
+        spectral_loss = penalty*tf.reduce_mean(tf.abs(spectrogram_true-spectrogram_pred))
+    #L2 LOSS
+    elif LOSS_TYPE =='L2':
+        spectral_loss = penalty*tf.reduce_mean((spectrogram_true - spectrogram_pred)**2)
+    #COSINE LOSS
+    elif LOSS_TYPE == 'COSINE':
+        spectral_loss = tf.losses.cosine_distance(spectrogram_true, spectrogram_pred, weights=1.0, axis=-1)
+    
+    return spectral_loss
+
+def summarize_compile(model: keras.Model):
+    model.summary(line_length=80, positions=[0.33, 0.65, 0.8, 1.0], show_trainable=True, expand_nested=True)
+    # Specify the training configuration (optimizer, loss, metrics)
+    model.compile(
+        optimizer=keras.optimizers.Adam(),  # Optimizer- Adam [14] optimizer
+        # Loss function to minimize
+        # @paper: Therefore, we converged on using sigmoid activations with binary cross entropy loss.
+        # loss=keras.losses.BinaryCrossentropy(),
+        loss=CustomLoss,
+        # List of metrics to monitor
+        metrics=[
+            # @paper: 1) Mean Percentile Rank?
+            # mean_percentile_rank,
+            # @paper: 2) Top-k mean accuracy based evaluation
+            top_k_mean_accuracy,
+            custom_spectral_loss,
+            # Extra Adding 3) spectroloss accuracy
+            # Extra Adding 4) combined
+            # @paper: 5) Mean Absolute Error based evaluation
+            keras.metrics.MeanAbsoluteError(),
+        ],
+    )
+
+def fit(
+    model: keras.Model,
+    x_train: np.ndarray,
+    y_train: np.ndarray,
+    x_val: np.ndarray,
+    y_val: np.ndarray,
+    batch_size: int = 16,
+    epochs: int = 200,
+) -> keras.Model:
+
+    # @paper:
+    # with a minibatch size of 16 for
+    # 100 epochs. The best weights for each model were set by
+    # employing an early stopping procedure.
+    logging.info("# Fit model on training data")
+    history = model.fit(
+        x_train,
+        y_train,
+        batch_size=batch_size,
+        epochs=epochs,
+        # @paper:
+        # Early stopping procedure:
+        # We pass some validation for
+        # monitoring validation loss and metrics
+        # at the end of each epoch
+        validation_data=(x_val, y_val),
+        verbose=0,
+    )
+
+    # The returned "history" object holds a record
+    # of the loss values and metric values during training
+    logging.info("\nhistory dict:", history.history)
+
+    return model
+
+
+def compare(target, prediction, params, precision=1, print_output=False):
+    if print_output and len(prediction) < 10:
+        print(prediction)
+        print("Pred: {}".format(np.round(prediction, decimals=2)))
+        print("PRnd: {}".format(np.round(prediction)))
+        print("Act : {}".format(target))
+        print("+" * 5)
+
+    pred: List[ParamValue] = params.decode(prediction)
+    act: List[ParamValue] = params.decode(target)
+    pred_index: List[int] = [np.array(p.encoding).argmax() for p in pred]
+    act_index: List[int] = [np.array(p.encoding).argmax() for p in act]
+    width = 8
+    names = "Parameter: "
+    act_s = "Actual:    "
+    pred_s = "Predicted: "
+    pred_i = "Pred. Indx:"
+    act_i = "Act. Index:"
+    diff_i = "Index Diff:"
+    for p in act:
+        names += p.name.rjust(width)[:width]
+        act_s += f"{p.value:>8.2f}"
+    for p in pred:
+        pred_s += f"{p.value:>8.2f}"
+    for p in pred_index:
+        pred_i += f"{p:>8}"
+    for p in act_index:
+        act_i += f"{p:>8}"
+    for i in range(len(act_index)):
+        diff = pred_index[i] - act_index[i]
+        diff_i += f"{diff:>8}"
+    exact = 0.0
+    close = 0.0
+    n_params = len(pred_index)
+    for i in range(n_params):
+        if pred_index[i] == act_index[i]:
+            exact = exact + 1.0
+        if abs(pred_index[i] - act_index[i]) <= precision:
+            close = close + 1.0
+    exact_ratio = exact / n_params
+    close_ratio = close / n_params
+    if print_output:
+        print(names)
+        print(act_s)
+        print(pred_s)
+        print(act_i)
+        print(pred_i)
+        print(diff_i)
+        print("-" * 30)
+    return exact_ratio, close_ratio
+
+
+def evaluate(
+    prediction: np.ndarray, x: np.ndarray, y: np.ndarray, params: ParameterSet,
+):
+
+    print("Prediction Shape: {}".format(prediction.shape))
+
+    num: int = x.shape[0]
+    correct: int = 0
+    correct_r: float = 0.0
+    close_r: float = 0.0
+    for i in range(num):
+        should_print = i < 5
+        exact, close = compare(
+            target=y[i],
+            prediction=prediction[i],
+            params=params,
+            print_output=should_print,
+        )
+        if exact == 1.0:
+            correct = correct + 1
+        correct_r += exact
+        close_r += close
+    summary = params.explain()
+    print(
+        "{} Parameters with {} levels (fixed: {})".format(
+            summary["n_variable"], summary["levels"], summary["n_fixed"]
+        )
+    )
+    print(
+        "Got {} out of {} ({:.1f}% perfect); Exact params: {:.1f}%, Close params: {:.1f}%".format(
+            correct,
+            num,
+            correct / num * 100,
+            correct_r / num * 100,
+            close_r / num * 100,
+        )
+    )
+
+
+def data_format_audio(audio: np.ndarray, data_format: str) -> np.ndarray:
+    # `(None, n_channel, n_freq, n_time)` if `'channels_first'`,
+    # `(None, n_freq, n_time, n_channel)` if `'channels_last'`,
+
+    if data_format == "channels_last":
+        audio = audio[np.newaxis, :, np.newaxis]
+    else:
+        audio = audio[np.newaxis, np.newaxis, :]
+
+    return audio
+
+
+"""
+Wrap up the whole training process in a standard function. Gets a callback
+to actually make the model, to keep it as flexible as possible.
+# Params:
+# - dataset_name (dataset name)
+# - model_name: (C1..C6,e2e)
+# - model_callback: function taking name,inputs,outputs,data_format and returning a Keras model
+# - epochs: int
+# - dataset_dir: place to find input data
+# - output_dir: place to put outputs
+# - parameters_file (override parameters filename)
+# - dataset_file (override dataset filename)
+# - data_format (channels_first or channels_last)
+# - run_name: to save this run as
+"""
+#LOSS TYPE FOR CUSTOM LOSS FUNCTION
+LOSS_TYPE = 'L1'
+SPECTRO_TYPE = 'spectro'
+PRINT = 1
+
+#DAWDREAMER EXPORT SETTINGS
+SAMPLE_RATE = 16384
+BUFFER_SIZE = 1024
+SYNTH_PLUGIN = "libTAL-NoiseMaker.so"
+
+ENGINE = daw.RenderEngine(SAMPLE_RATE, BUFFER_SIZE)
+SYNTH = ENGINE.make_plugin_processor("my_synth", SYNTH_PLUGIN)
+SYNTH.add_midi_note(40, 127, 0, 0.8)
+
+with open('plugin_config/TAL-NoiseMaker-config.json') as f:
+    data = json.load(f)
+
+dico=[]
+# Extract the key ID from the JSON data
+key_id = data['parameters']
+for param in key_id:
+    dico.append(param['id'])
+
+DICO=dico
+
+def train_model(
+    # Main options
+    dataset_name: str,
+    model_name: str,
+    epochs: int,
+    model_callback: Callable[[str, int, int, str], keras.Model],
+    dataset_dir: str,
+    output_dir: str,  # Directory names
+    dataset_file: str = None,
+    parameters_file: str = None,
+    run_name: str = None,
+    data_format: str = "channels_last",
+    save_best: bool = True,
+    resume: bool = False,
+    checkpoint: bool = True,
+    model_type: str = "STFT",
+):
+
+    tf.config.run_functions_eagerly(True)
+    # tf.data.experimental.enable_debug_mode()
+    time_generated = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+    if not dataset_file:
+        dataset_file = (
+            os.getcwd() + "/" + dataset_dir + "/" + dataset_name + "_data.hdf5"
+        )
+    if not parameters_file:
+        parameters_file = (
+            os.getcwd() + "/" + dataset_dir + "/" + dataset_name + "_params.pckl"
+        )
+    if not run_name:
+        run_name = dataset_name + "_" + model_name
+
+    model_file = f"{output_dir}/model/{run_name}_{time_generated}"
+    if not os.path.exists(model_file):
+        os.makedirs(model_file)
+    best_model_file = f"{output_dir}/best_checkpoint/{run_name}_best_{time_generated}"
+    if not os.path.exists(best_model_file):
+        os.makedirs(best_model_file)
+    if resume: 
+        
+        # checkpoint_model_file = f"{output_dir}/{run_name}_checkpoint_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
+        # history_file = f"{output_dir}/{run_name}_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
+        checkpoint_model_file = f"{output_dir}/history/InverSynth_C6XL_checkpoint_20231201-103344"
+        history_file = f"{output_dir}/checkpoints/InverSynth_C6XL_20231201-103344"
+    else:
+        os.makedirs(f"{output_dir}/history", exist_ok=True)
+        os.makedirs(f"{output_dir}/checkpoints", exist_ok=True)
+        history_file = f"{output_dir}/history/{run_name}_{time_generated}"
+        checkpoint_model_file = f"{output_dir}/checkpoints/{run_name}_checkpoint_{time_generated}"
+            
+    history_graph_file = f"{output_dir}/{run_name}.pdf"
+    print(tf.config.list_physical_devices('GPU'))
+    gpu_avail = len(tf.config.list_physical_devices('GPU'))  # True/False
+    cuda_gpu_avail = len(tf.config.list_physical_devices('GPU'))  # True/False
+
+    print("+" * 30)
+    print(f"++ {run_name}")
+    print(
+        f"Running model: {model_name} on dataset {dataset_file} (parameters {parameters_file}) for {epochs} epochs"
+    )
+    print(f"Saving model in {output_dir} as {model_file}")
+    print(f"Saving history as {history_file}")
+    print(f"GPU: {gpu_avail}, with CUDA: {cuda_gpu_avail}")
+    print("+" * 30)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Get training and validation generators
+    params = {"data_file": dataset_file, "batch_size": 64, "shuffle": True}
+    training_generator = SoundDataGenerator(first=0.8, **params)
+    validation_generator = SoundDataGenerator(last=0.2, **params)
+    n_samples = training_generator.get_audio_length()
+    print(f"get_audio_length: {n_samples}")
+    n_outputs = training_generator.get_label_size()
+
+    # set keras image_data_format
+    # NOTE: on CPU only `channels_last` is supported
+    physical_devices = tf.config.list_physical_devices('GPU')
+   
+    keras.backend.set_image_data_format(data_format)
+
+    model: keras.Model = None
+    if resume and os.path.exists(checkpoint_model_file):
+        history = pd.read_csv(history_file)
+        # Note - its zero indexed in the file, but 1 indexed in the display
+        initial_epoch: int = max(history.iloc[:, 0]) + 1
+        # epochs:int = initial_epoch
+        print(
+            f"Resuming from model file: {checkpoint_model_file} after epoch {initial_epoch}"
+        )
+        model = keras.models.load_model(
+            checkpoint_model_file
+            ,
+            custom_objects={"top_k_mean_accuracy": top_k_mean_accuracy, "Spectrogram" : Spectrogram,
+                            "custom_spectral_loss": custom_spectral_loss, "CustomLoss": CustomLoss
+                            },
+        )
+    else:
+        model = model_callback(
+            model_name=model_name,
+            inputs=n_samples,
+            outputs=n_outputs,
+            data_format=data_format,
+        )
+        # keras.utils.plot_model(model, to_file='model.png', show_shapes=True, show_layer_activations=True)
+        # Summarize and compile the model
+        summarize_compile(model)
+        initial_epoch = 0
+        open(history_file, "w").close()
+
+    callbacks = []
+    best_callback = keras.callbacks.ModelCheckpoint(
+        filepath=best_model_file,
+        save_weights_only=False,
+        save_best_only=True,
+        verbose=1,
+    )
+    checkpoint_callback = keras.callbacks.ModelCheckpoint(
+        filepath=checkpoint_model_file,
+        save_weights_only=False,
+        save_best_only=False,
+        verbose=1,
+    )
+    os.makedirs(f"{output_dir}/logs", exist_ok=True)
+    log_dir = f"{output_dir}/logs/" + time_generated
+    tensorboard_callback = keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, write_graph=True, write_images=True, profile_batch = '500,520')
+
+    if save_best:
+        callbacks.append(best_callback)
+    if checkpoint:
+        callbacks.append(checkpoint_callback)
+    callbacks.append(tensorboard_callback)
+    callbacks.append(CSVLogger(history_file, append=True))
+    callbacks.append(Weight_trans(weight_var, "log3" ,epochs))
+        # Parameter data - needed for decoding!
+    
+    # Fit the model
+    history = None
+    try:
+        history =   model.fit(
+            x=training_generator,
+            validation_data=validation_generator,
+            epochs=epochs,
+            callbacks=callbacks,
+            initial_epoch=initial_epoch,
+            verbose=1,  # https://github.com/tensorflow/tensorflow/issues/38064
+        )
+    except Exception as e:
+        print(f"Something went wrong during `model.fit`: {e}")
+
+    # Save model
+    model.save(model_file)
+
+    # Save history
+    if history and not resume:
+        try:
+            hist_df = pd.DataFrame(history.history)
+            try:
+                fig = hist_df.plot(subplots=True, figsize=(8, 25))
+                fig[0].get_figure().savefig(history_graph_file)
+            except Exception as e:
+                print("Couldn't create history graph")
+                print(e)
+
+        except Exception as e:
+            tf.print("Couldn't save history")
+            print(e)
+
+    # evaluate prediction on random sample from validation set
+    # Parameter data - needed for decoding!
+    with open(parameters_file, "rb") as f:
+        parameters: ParameterSet = load(f)
+
+    # Shuffle data
+    validation_generator.on_epoch_end()
+    X, y = validation_generator.__getitem__(0)
+    X.reshape((X.__len__(), 1, 16384))
+
+    # if model_type == "STFT":
+    #     # stft expects shape (channel, sample_rate)
+    #     X = np.moveaxis(X, 1, -1)
+    prediction: np.ndarray = model.predict(X)
+    evaluate(prediction, X, y, parameters)
+    
+    print("++++" * 5)
+    print("Pushing to trained model")
+    print("++++" * 5)
+    
+    Valid=False
+    while Valid==False:
+        file = namefile = input("Enter .wav test file path: ")
+        if os.path.exists(file):
+            Valid=True
+        else:
+            print("File Path invalid, try again ")
+        
+    newpred = model.predict(audio_importer(str(f'{namefile}')))
+    predlist: List[ParamValue] = parameters.decode(newpred[0])
+    df = pd.DataFrame(predlist) 
+    
+    print(df)
+    df = df.drop(['encoding'], axis=1)
+    # saving the dataframe
+    if not os.path.exists(str(f'output/wav_inferred')):
+        os.makedirs(str(f'output/wav_inferred'))
+    head, tail = os.path.split(str(f'{namefile}'))
+    print("Outputting CSV config in " + str(f'output/wav_inferred'))
+    df.to_csv(str(f'output/wav_inferred/{tail}.csv'))
+    #export(prediction, X, y, parameters)
+    # Loop through the rows of the DataFrame
+    i = 0
+    for values in df['value'].values:
+        # Set parameters using DataFrame values
+        SYNTH.set_parameter(DICO[i],values)
+        # (MIDI note, velocity, start, duration)
+        i += 1
+    #Setting volume to 0.9
+    SYNTH.set_parameter(1, 0.9)
+    # Set up the processing graph
+    graph = [
+        # synth takes no inputs, so we give an empty list.
+        (SYNTH, []),
+    ]
+
+    ENGINE.load_graph(graph)
+    ENGINE.render(1)
+    data = ENGINE.get_audio()
+    try:
+        data = librosa.to_mono(data).transpose()
+    except:
+        tf.print("ERROR" * 100)
+        df = df.fillna(0)
+        data = df.to_numpy()
+        data = librosa.to_mono(data).transpose()
+        tf.print("crashed, nan in generation")
+        synth_params = dict(SYNTH.get_patch())
+        print(synth_params)
+    
+    df = pd.DataFrame(data)
+        
+        # penalty=1000000
+        # df = pd.DataFrame(data)
+        # df = df.fillna(0)
+        # data = df.to_numpy()
+
+          
+    wavfile.write(str(f'output/wav_inferred/gen_{tail}.wav'), SAMPLE_RATE, data)
+        
+def generate_audio(df_params):
+
+    
+    # Loop through the rows of the DataFrame
+    i = 0
+    penalty=1
+    for param in df_params:
+        # Set parameters using DataFrame values
+        SYNTH.set_parameter(DICO[i], param.value)
+        # (MIDI note, velocity, start, duration)
+        i += 1
+    # Set up the processing graph
+    graph = [
+        # synth takes no inputs, so we give an empty list.
+        (SYNTH, []),
+    ]
+
+    ENGINE.load_graph(graph)
+    ENGINE.render(1)
+    data = ENGINE.get_audio()
+    if np.isnan(data).any():
+        
+        # df = pd.DataFrame(data)
+        # df = df.fillna(0)
+        # data = df.to_numpy()
+
+        tf.print("crashed, nan in generation")
+        synth_params = dict(SYNTH.get_patch())
+        print(synth_params)
+    try:
+        data = librosa.to_mono(data).transpose()
+        if(librosa.util.valid_audio(data)):
+            result = np.array(data)
+            return result, penalty
+    except:
+        tf.print("crashed, nan in generation")
+        raise("Nan in generation, crashed")
+  
+    
+    

back/models/common/architectures.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass
+
+
+# Model architectures
+@dataclass
+class C:
+    filters: int
+    window_size: tuple
+    strides: tuple
+    activation: str = "relu"
+
+
+"""Conv 1 (2 Layers)"""
+c1: C = C(38, (13, 26), (13, 26))
+c1_layers: list = [c1]
+
+"""Conv 2 (3 Layers)"""
+c2_layers: list = [C(35, (6, 7), (5, 6)), C(87, (6, 9), (5, 8))]
+
+"""Conv 3 (4 Layers)"""
+c3_layers: list = [C(32, (4, 5), (3, 4)), C(98, (4, 6), (3, 5)), C(128, (4, 6), (3, 5))]
+
+"""Conv 4 (5 Layers)"""
+c4_layers: list = [
+    C(32, (3, 4), (2, 3)),
+    C(65, (3, 4), (2, 3)),
+    C(105, (3, 4), (2, 3)),
+    C(128, (4, 5), (3, 4)),
+]
+
+"""Conv 5 (6 Layers)"""
+c5_layers: list = [
+    C(32, (3, 3), (2, 2)),
+    C(98, (3, 3), (2, 2)),
+    C(128, (3, 4), (2, 3)),
+    C(128, (3, 5), (2, 4)),
+    C(128, (3, 3), (2, 2)),
+]
+
+"""Conv 6 (7 Layers)"""
+c6_layers: list = [
+    C(32, (3, 3), (2, 2)),
+    C(71, (3, 3), (2, 2)),
+    C(128, (3, 4), (2, 3)),
+    C(128, (3, 3), (2, 2)),
+    C(128, (3, 3), (2, 2)),
+    C(128, (3, 3), (1, 2)),
+]
+
+"""Conv 6XL, 7 Layers"""
+c6XL_layers: list = [
+    C(64, (3, 3), (2, 2)),
+    C(128, (3, 3), (2, 2)),
+    C(128, (3, 4), (2, 3)),
+    C(128, (3, 3), (2, 2)),
+    C(256, (3, 3), (2, 2)),
+    C(256, (3, 3), (1, 2)),
+]
+
+
+layers_map: dict = {
+    "C1": c1_layers,
+    "C2": c2_layers,
+    "C3": c3_layers,
+    "C4": c4_layers,
+    "C5": c5_layers,
+    "C6": c6_layers,
+    "C6XL": c6XL_layers,
+}

back/models/common/data_generator.py

@@ -0,0 +1,126 @@
+import h5py
+import numpy as np
+from scipy.io import wavfile
+from tensorflow import keras
+
+
+class SoundDataGenerator(keras.utils.Sequence):
+    "Generates data for Keras"
+
+    def __init__(
+        self,
+        data_file=None,
+        batch_size=32,
+        n_samps=16384,
+        shuffle=True,
+        last: float = 0.0,
+        first: float = 0.0,
+        channels_last=False,
+        for_autoencoder=False,
+    ):
+        "Initialization"
+        self.dim = (1, n_samps)
+        self.batch_size = batch_size
+        self.shuffle = shuffle
+        self.data_file = data_file
+        self.n_channels = 1
+        self.for_autoencoder = for_autoencoder
+        # For the E2E model, need to return channels last?
+        if channels_last:
+            self.expand_axis = 2
+        else:
+            self.expand_axis = 1
+
+        database = h5py.File(data_file, "r")
+
+        self.database = database
+
+        self.n_samps = self.read_file(0).shape[0]
+        print("N Samps in audio data: {}".format(self.n_samps))
+
+        # set up list of IDs from data files
+        n_points = len(database["files"])
+        self.list_IDs = range(len(database["files"]))
+
+        print(f"Number of examples in dataset: {len(self.list_IDs)}")
+        slice: int = 0
+        if last > 0.0:
+            slice = int(n_points * (1 - last))
+            self.list_IDs = self.list_IDs[slice:]
+            print(f"Taking Last N points: {len(self.list_IDs)}")
+        elif first > 0.0:
+            slice = int(n_points * first)
+            self.list_IDs = self.list_IDs[:slice]
+            print(f"Taking First N points: {len(self.list_IDs)}")
+
+        # set up label size from data files
+        self.label_size = len(database["labels"][0])
+        self.on_epoch_end()
+
+    def get_audio_length(self):
+        return self.n_samps
+
+    def get_label_size(self):
+        return self.label_size
+
+    def __len__(self):
+        "Denotes the number of batches per epoch"
+        return int(np.floor(len(self.list_IDs) / self.batch_size))
+
+    def __getitem__(self, index):
+        "Generate one batch of data"
+        # Generate indexes of the batch
+        indexes = self.indexes[index * self.batch_size : (index + 1) * self.batch_size]
+
+        # Find list of IDs
+        # list_IDs_temp = [self.list_IDs[k] for k in indexes]
+
+        # Generate data
+        X, y = self.__data_generation(indexes)
+
+        # print("Returning data! Got X: {}, y: {}".format(X.shape,y.shape))
+        return X, y
+
+    def on_epoch_end(self):
+        "Updates indexes after each epoch"
+        self.indexes = np.arange(len(self.list_IDs))
+        if self.shuffle is True:
+            np.random.shuffle(self.indexes)
+
+    # Think this makes things worse - fills up memory
+    # @lru_cache(maxsize=150000)
+    def read_file(self, index):
+        filename = self.database["files"][index]
+        fs, data = wavfile.read(filename)
+        return data
+
+    def __data_generation(self, list_IDs_temp):
+        # X : (n_samples, *dim, n_channels)
+        "Generates data containing batch_size samples"
+        # Initialization    
+        # X = np.empty((self.batch_size, *self.dim))
+        # y = np.empty((self.batch_size), dtype=int)
+
+        # Generate data
+        X = []
+        y = []
+        for i in list_IDs_temp:
+            # Read labels
+            y.append(self.database["labels"][i])
+            # Load soundfile data
+            data = self.read_file(i)
+            if data.shape[0] > self.n_samps:
+                print(
+                    "Warning - too many samples: {} > {}".format(
+                        data.shape[0], self.n_samps
+                    )
+                )
+            X.append(data[: self.n_samps])
+        Xd = np.expand_dims(np.vstack(X), axis=1)
+        # Xd = Xd.flatten()
+        Xd = Xd.reshape((X.__len__(), 1, self.n_samps))
+        yd = np.vstack(y)
+
+        if self.for_autoencoder:
+            return yd, yd
+        return Xd, yd

back/models/comparison.py

@@ -0,0 +1,144 @@
+import json
+import os
+import pickle
+import re
+
+import h5py
+import numpy as np
+from scipy.io import wavfile
+from scipy.io.wavfile import write as write_wav
+from tensorflow import keras
+
+from generators.generator import InverSynthGenerator, SoundGenerator, VSTGenerator
+from generators.parameters import ParameterSet
+
+"""
+This module generates comparisons - takes the original sound + params,
+then generates a file with the predicted parameters
+"""
+
+
+def compare(
+    model: keras.Model,
+    generator: SoundGenerator,
+    parameters: ParameterSet,
+    orig_file: str,
+    output_dir: str,
+    orig_params,
+    length: float,
+    sample_rate: int,
+    extra: dict = {},
+):
+    # (copy original file if given)
+    base_filename = orig_file.replace(".wav", "")
+    base_filename = re.sub(r".*/", "", base_filename)
+    copy_file: str = f"{output_dir}/{base_filename}_copy.wav"
+    regen_file: str = f"{output_dir}/{base_filename}_duplicate.wav"
+    reconstruct_file: str = f"{output_dir}/{base_filename}_reconstruct.wav"
+    print(f"Creating copy as {copy_file}")
+
+    # Load the wave file
+    fs, data = wavfile.read(orig_file)
+    # Copy original file to make sure
+    write_wav(copy_file, sample_rate, data)
+
+    # Decode original params, and regenerate output (make sure its correct)
+    orig = parameters.encoding_to_settings(orig_params)
+    generator.generate(orig, regen_file, length, sample_rate, extra)
+
+    # Run the wavefile into the model for prediction
+    X = [data]
+    Xd = np.expand_dims(np.vstack(X), axis=2)
+    # Get encoded parameters out of model
+    result = model.predict(Xd)[0]
+
+    # Decode prediction, and reconstruct output
+    predicted = parameters.encoding_to_settings(result)
+    generator.generate(predicted, reconstruct_file, length, sample_rate, extra)
+
+
+def run_comparison(
+    model: keras.Model,
+    generator: SoundGenerator,
+    run_name: str,
+    indices=None,
+    num_samples=10,
+    data_dir="./test_datasets",
+    output_dir="./comparison",
+    length=1.0,
+    sample_rate=16384,
+    shuffle=True,
+    extra={},
+):
+    # Figure out data file and params file from run name
+    data_file = f"{data_dir}/{run_name}_data.hdf5"
+    parameters_file = f"{data_dir}/{run_name}_params.pckl"
+    print(f"Reading parameters from {parameters_file}")
+    parameters = pickle.load(open(parameters_file, "rb"))
+
+    output_dir = f"{output_dir}/{run_name}/"
+    os.makedirs(output_dir, exist_ok=True)
+
+    database = h5py.File(data_file, "r")
+
+    if not indices:
+        ids = np.array(range(len(database["files"])))
+        if shuffle:
+            np.random.shuffle(ids)
+        indices = ids[0:num_samples]
+
+    # filename
+    for i in indices:
+        print("Looking at index: {}".format(i))
+        filename = database["files"][i]
+        labels = database["labels"][i]
+        compare(
+            model=model,
+            generator=generator,
+            parameters=parameters,
+            orig_file=filename,
+            output_dir=output_dir,
+            orig_params=labels,
+            length=length,
+            sample_rate=sample_rate,
+            extra=extra,
+        )
+    # Generate
+
+
+if __name__ == "__main__":
+
+    note_length = 0.8
+    sample_rate = 16384
+
+    lokomotiv = True
+    fm = True
+
+    if lokomotiv:
+        from generators.vst_generator import *
+
+        run_name = "lokomotiv_full"
+        model_file = "output/lokomotiv_full_e2e_best.h5"
+        plugin = "/Library/Audio/Plug-Ins/VST/Lokomotiv.vst"
+        config_file = "plugin_config/lokomotiv.json"
+        generator = VSTGenerator(vst=plugin, sample_rate=sample_rate)
+        with open(config_file, "r") as f:
+            config = json.load(f)
+
+        model = keras.models.load_model(model_file)
+        run_comparison(
+            model,
+            generator,
+            run_name,
+            num_samples=100,
+            extra={"note_length": note_length, "config": config},
+        )
+
+    if fm:
+        from generators.fm_generator import *
+
+        run_name = "inversynth_full"
+        model_file = "output/inversynth_full_e2e_best.h5"
+        generator = InverSynthGenerator()
+        model = keras.models.load_model(model_file)
+        run_comparison(model, generator, run_name, num_samples=100)

back/models/convert_to_preset.py

@@ -0,0 +1,149 @@
+import csv;
+import xml.etree.ElementTree as ET
+
+def convert_csv_to_preset(path: str, output_path: str):
+
+    daw_to_preset_og = {
+        'Master Volume': 'volume',
+        'Filter Type': 'filtertype',
+        'Filter Cutoff': 'cutoff',
+        'Filter Resonance': 'resonance',
+        'Filter Keyfollow': 'keyfollow',
+        'Filter Contour': 'filtercontour',
+        'Filter Attack': 'filterattack',
+        'Filter Decay': 'filterdecay',
+        'Filter Sustain': 'filtersustain',
+        'Filter Release': 'filterrelease',
+        'Amp Attack': 'ampattack',
+        'Amp Decay': 'ampdecay',
+        'Amp Sustain': 'ampsustain',
+        'Amp Release': 'amprelease',
+        'Osc 1 Volume': 'osc1volume',
+        'Osc 2 Volume': 'osc2volume',
+        'Osc 3 Volume': 'osc3volume',
+        'Osc Mastertune': 'oscmastertune',
+        'Osc 1 Tune': 'osc1tune',
+        'Osc 2 Tune': 'osc2tune',
+        'Osc 1 Fine Tune': 'osc1finetune',
+        'Osc 2 Fine Tune': 'osc2finetune',
+        'Osc 1 Waveform': 'osc1waveform',
+        'Osc 2 Waveform': 'osc2waveform',
+        'Osc Sync': 'oscsync',
+        'Lfo 1 Waveform': 'lfo1waveform',
+        'Lfo 2 Waveform': 'lfo2waveform',
+        'Lfo 1 Rate': 'lfo1rate',
+        'Lfo 2 Rate': 'lfo2rate',
+        'Lfo 1 Amount': 'lfo1amount',
+        'Lfo 2 Amount': 'lfo2amount',
+        'Lfo 1 Destination': 'lfo1destination',
+        'Lfo 2 Destination': 'lfo2destination',
+        'Lfo 1 Phase': 'lfo1phase',
+        'Lfo 2 Phase': 'lfo2phase',
+        'Osc 2 FM': 'osc2fm',
+        'Osc 2 Phase': 'osc2phase',
+        'Osc 1 PW': 'osc1pw',
+        'Osc 1 Phase': 'osc1phase',
+        'Transpose': 'transpose',
+        'Free Ad Attack': 'freeadattack',
+        'Free Ad Decay': 'freeaddecay',
+        'Free Ad Amount': 'freeadamount',
+        'Free Ad Destination': 'freeaddestination',
+        'Lfo 1 Sync': 'lfo1sync',
+        'Lfo 1 Keytrigger': 'lfo1keytrigger',
+        'Lfo 2 Sync': 'lfo2sync',
+        'Lfo 2 Keytrigger': 'lfo2keytrigger',
+        'Portamento Amount': 'portamento',
+        'Portamento Mode': 'portamentomode',
+        'Voices': 'voices',
+        'Velocity Volume': 'velocityvolume',
+        'Velocity Contour': 'velocitycontour',
+        'Velocity Filter': 'velocitycutoff',
+        'Pitchwheel Cutoff': 'pitchwheelcutoff',
+        'Pitchwheel Pitch': 'pitchwheelpitch',
+        'Ringmodulation': 'ringmodulation',
+        'Chorus 1 Enable': 'chorus1enable',
+        'Chorus 2 Enable': 'chorus2enable',
+        'Reverb Wet': 'reverbwet',
+        'Reverb Decay': 'reverbdecay',
+        'Reverb Pre Delay': 'reverbpredelay',
+        'Reverb High Cut': 'reverbhighcut',
+        'Reverb Low Cut': 'reverblowcut',
+        'Osc Bitcrusher': 'oscbitcrusher',
+        'Master High Pass': 'highpass',
+        'Master Detune': 'detune',
+        'Vintage Noise': 'vintagenoise',
+        'Envelope Destination': 'envelopeeditordest1',
+        'Envelope Speed': 'envelopeeditorspeed',
+        'Envelope Amount': 'envelopeeditoramount',
+        'Envelope One Shot Mode': 'envelopeoneshot',
+        'Envelope Fix Tempo': 'envelopefixtempo',
+        'Filter Drive': 'filterdrive',
+        'Delay Wet': 'delaywet',
+        'Delay Time': 'delaytime',
+        'Delay Sync': 'delaysync',
+        'Delay x2 L': 'delayfactorl',
+        'Delay x2 R': 'delayfactorr',
+        'Delay High Shelf': 'delayhighshelf',
+        'Delay Low Shelf': 'delaylowshelf',
+        'Delay Feedback': 'delayfeedback',
+    }
+
+    daw_to_preset = {v: k for k, v in daw_to_preset_og.items()}
+
+    # Read CSV data from file
+    with open(path, 'r') as csv_file:
+        csv_reader = csv.DictReader(csv_file)
+        csv_data = list(csv_reader)
+
+    for entry in csv_data:
+        parameter_name = entry['name']
+        parameter_value_str = entry['value']
+        
+       
+        # Check if the name needs mapping
+        if parameter_name in daw_to_preset_og:
+            xml_key = daw_to_preset_og[parameter_name]
+
+        # Check if the value is numeric
+        try:
+            parameter_value = float(parameter_value_str)
+        except ValueError:
+            print(f"Skipping non-numeric value for parameter {parameter_name}: {parameter_value_str}")
+            continue
+
+        if xml_key in daw_to_preset:
+            # Update the corresponding value in the XML dictionary
+            daw_to_preset[xml_key] = parameter_value
+
+    print(daw_to_preset)
+
+    # Check for invalid float values and remove them from the dictionary
+    invalid_values = [key for key, value in daw_to_preset.items() if not isinstance(value, float)]
+    for key in invalid_values:
+        print(f"Removing attribute {key} from daw_to_preset due to invalid float value.")
+        daw_to_preset[key] = 0.0
+    # Print the updated XML dictionary
+    print(daw_to_preset)
+
+    # Generate XML
+    root = ET.Element('tal', curprogram="0", version="1.7", presetName="CH Chordionator III FN",
+                    path="Factory Presets/CHORD/CH Chordionator III FN.noisemakerpreset")
+    programs = ET.SubElement(root, 'programs')
+    program = ET.SubElement(programs, 'program', programname="CH Chordionator III FN", unknown="0.5", volume="0.5")
+    # Add parameters to the XML inside the single <program> element
+    for param_name, param_value in daw_to_preset.items():
+        program.set(param_name, str(param_value))
+        
+        
+    ET.SubElement(root, 'midimap')
+    # Create an ElementTree object
+
+    tree = ET.ElementTree(root)
+
+    # Save the XML to a file
+    output_xml_path = output_path
+    tree.write(output_xml_path)
+
+    print(f"XML file written to {output_xml_path}")
+    
+    return output_xml_path

back/models/importer_audio.py

@@ -0,0 +1,23 @@
+import librosa.core.audio
+import numpy as np
+from scipy.io import wavfile
+
+
+def audio_importer(filename):
+    # X : (n_samples, *dim, n_channels)
+    "Generates data containing batch_size samples"
+
+    # Generate data
+    X = []
+    # Load soundfile data
+    data, r = librosa.core.audio.load(filename, sr=16384, mono=True, duration=1.0)
+    n_samps = data.shape[0]
+    print("N Samps in audio data: {}".format(n_samps))
+    X.append(data[: n_samps])
+    Xd = np.expand_dims(np.vstack(X), axis=1)
+    Xd = Xd.reshape((X.__len__(), 1, n_samps))
+
+    return Xd
+
+if __name__ == "__main__":
+    audio_importer("C:/Users/yderre/Downloads/inver-synth-master/inver-synth-master/test_waves/InverSynth/InverSynth_00006.wav")

back/models/launch.py

@@ -0,0 +1,518 @@
+import datetime
+import json
+import os
+from pickle import load
+from typing import Callable, List
+import librosa
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from tensorflow import keras
+from keras import backend as K
+from kapre.time_frequency import Spectrogram
+from models.convert_to_preset import convert_csv_to_preset
+from models.importer_audio import audio_importer
+import dawdreamer as daw
+from scipy.io import wavfile
+import librosa
+
+
+from generators.parameters import ParameterSet, ParamValue
+
+
+weight_var = K.variable(0.0)
+
+
+class Weight_trans(keras.callbacks.Callback):
+    def __init__(self, weight_var, transition, epochs):
+        self.alpha = weight_var
+        self.transition = transition
+        self.epochs = epochs
+
+    def on_epoch_end(self, epoch, logs={}):
+        if epoch > 680:
+            if self.transition == "linear":
+                K.set_value(self.alpha, ((epoch) / (self.epochs) - 0.617) * 0.00001)
+                tf.print(f"new weight {weight_var.numpy()}")
+            if self.transition == "linear2":
+                K.set_value(self.alpha, (1.5625 * epoch - 1.0625) * 0.00001)
+                tf.print(f"new weight {weight_var.numpy()}")
+            if self.transition == "log":
+                K.set_value(
+                    self.alpha,
+                    (
+                        1
+                        - (tf.math.log(epoch * 0.001 - 0.67285) / tf.math.log(0.0005))
+                        - 0.35
+                    )
+                    * 0.00001,
+                )
+                tf.print("log")
+            if self.transition == "log2":
+                K.set_value(
+                    self.alpha,
+                    (
+                        1
+                        - (tf.math.log(epoch * 0.001 - 0.6575) / tf.math.log(0.0005))
+                        - 0.5
+                    )
+                    * 0.00001,
+                )
+                tf.print("log")
+            if self.transition == "log3":
+                K.set_value(
+                    self.alpha,
+                    (
+                        1
+                        - (
+                            tf.math.log(epoch * 0.001 - 0.67978)
+                            / tf.math.log(0.00000005)
+                        )
+                        - 0.5
+                    )
+                    * 0.00001,
+                )
+                tf.print("log")
+            if self.transition == "square":
+                K.set_value(self.alpha, 4.1 * tf.pow(epoch * 0.001 - 0.65, 2) + 0.002)
+                print("exp")
+            if self.transition == "quad":
+                K.set_value(self.alpha, 33 * tf.pow(epoch * 0.001 - 0.65, 4) + 0.002)
+                print("quad")
+
+
+"""Model Utils"""
+
+
+def mean_percentile_rank(y_true, y_pred, k=5):
+    """
+    @paper
+    The first evaluation measure is the Mean Percentile Rank
+    (MPR) which is computed per synthesizer parameter.
+    """
+    # TODO
+
+
+def top_k_mean_accuracy(y_true, y_pred, k=5):
+    """
+    @ paper
+    The top-k mean accuracy is obtained by computing the top-k
+    accuracy for each test example and then taking the mean across
+    all examples. In the same manner as done in the MPR analysis,
+    we compute the top-k mean accuracy per synthesizer
+    parameter for 𝑘 = 1, ... ,5.
+    """
+    # TODO: per parameter?
+    original_shape = tf.shape(y_true)
+    y_true = tf.reshape(y_true, (-1, tf.shape(y_true)[-1]))
+    y_pred = tf.reshape(y_pred, (-1, tf.shape(y_pred)[-1]))
+    top_k = K.in_top_k(y_pred, tf.cast(tf.argmax(y_true, axis=-1), "int32"), k)
+    correct_pred = tf.reshape(top_k, original_shape[:-1])
+    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))
+
+
+@tf.function
+def CustomLoss(y_true, y_pred):
+    bce = tf.keras.losses.BinaryCrossentropy()
+    weights = custom_spectral_loss(y_true, y_pred)
+    weight_shift = (1 - weight_var.numpy()) + (weight_var.numpy() * weights.numpy())
+    # tf.print(f"New weight is {weight_shift}")
+    loss = bce(y_true, y_pred, sample_weight=weight_shift)
+    return loss
+
+
+@tf.function
+def custom_spectral_loss(y_true, y_pred):
+    # tf.print("After compiling model :",tf.executing_eagerly())
+
+    y_true = tf.reshape(y_true, (-1, tf.shape(y_true)[-1]))
+    y_pred = tf.reshape(y_pred, (-1, tf.shape(y_pred)[-1]))
+
+    # Assuming y_true and y_pred contain parameters for audio synthesis
+    # Extract parameters from y_true and y_pred
+    with open("test_datasets/InverSynth_params.pckl", "rb") as f:
+        parameters: ParameterSet = load(f)
+
+    predlist_true: List[ParamValue] = parameters.decode(y_true[0])
+
+    predlist_pred: List[ParamValue] = parameters.decode(y_pred[0])
+
+    # Convert parameter lists to DataFrames
+    # Generate audio from parameters
+    audio_true, penalty = generate_audio(predlist_true)
+    audio_pred, penalty = generate_audio(predlist_pred)
+
+    # Compute spectrogram
+    if SPECTRO_TYPE == "spectro":
+        spectrogram_true = tf.math.abs(
+            tf.signal.stft(audio_true, frame_length=1024, frame_step=512)
+        )
+        spectrogram_pred = tf.math.abs(
+            tf.signal.stft(audio_pred, frame_length=1024, frame_step=512)
+        )
+    elif SPECTRO_TYPE == "qtrans":
+        spectrogram_true = librosa.amplitude_to_db(
+            librosa.cqt(audio_true, sr=SAMPLE_RATE, hop_length=128), ref=np.max
+        )
+        spectrogram_pred = librosa.amplitude_to_db(
+            librosa.cqt(audio_pred, sr=SAMPLE_RATE, hop_length=128), ref=np.max
+        )
+    elif SPECTRO_TYPE == "mel":
+        mel_spect = librosa.feature.melspectrogram(
+            audio_true, sr=SAMPLE_RATE, n_fft=2048, hop_length=1024
+        )
+        spectrogram_true = librosa.power_to_db(mel_spect, ref=np.max)
+        mel_spect = librosa.feature.melspectrogram(
+            audio_pred, sr=SAMPLE_RATE, n_fft=2048, hop_length=1024
+        )
+        spectrogram_pred = librosa.power_to_db(mel_spect, ref=np.max)
+    # L1 LOSS
+    if LOSS_TYPE == "L1":
+        spectral_loss = penalty * tf.reduce_mean(
+            tf.abs(spectrogram_true - spectrogram_pred)
+        )
+    # L2 LOSS
+    elif LOSS_TYPE == "L2":
+        spectral_loss = penalty * tf.reduce_mean(
+            (spectrogram_true - spectrogram_pred) ** 2
+        )
+    # COSINE LOSS
+    elif LOSS_TYPE == "COSINE":
+        spectral_loss = tf.losses.cosine_distance(
+            spectrogram_true, spectrogram_pred, weights=1.0, axis=-1
+        )
+
+    return spectral_loss
+
+
+def compare(target, prediction, params, precision=1, print_output=False):
+    if print_output and len(prediction) < 10:
+        print(prediction)
+        print("Pred: {}".format(np.round(prediction, decimals=2)))
+        print("PRnd: {}".format(np.round(prediction)))
+        print("Act : {}".format(target))
+        print("+" * 5)
+
+    pred: List[ParamValue] = params.decode(prediction)
+    act: List[ParamValue] = params.decode(target)
+    pred_index: List[int] = [np.array(p.encoding).argmax() for p in pred]
+    act_index: List[int] = [np.array(p.encoding).argmax() for p in act]
+    width = 8
+    names = "Parameter: "
+    act_s = "Actual:    "
+    pred_s = "Predicted: "
+    pred_i = "Pred. Indx:"
+    act_i = "Act. Index:"
+    diff_i = "Index Diff:"
+    for p in act:
+        names += p.name.rjust(width)[:width]
+        act_s += f"{p.value:>8.2f}"
+    for p in pred:
+        pred_s += f"{p.value:>8.2f}"
+    for p in pred_index:
+        pred_i += f"{p:>8}"
+    for p in act_index:
+        act_i += f"{p:>8}"
+    for i in range(len(act_index)):
+        diff = pred_index[i] - act_index[i]
+        diff_i += f"{diff:>8}"
+    exact = 0.0
+    close = 0.0
+    n_params = len(pred_index)
+    for i in range(n_params):
+        if pred_index[i] == act_index[i]:
+            exact = exact + 1.0
+        if abs(pred_index[i] - act_index[i]) <= precision:
+            close = close + 1.0
+    exact_ratio = exact / n_params
+    close_ratio = close / n_params
+    if print_output:
+        print(names)
+        print(act_s)
+        print(pred_s)
+        print(act_i)
+        print(pred_i)
+        print(diff_i)
+        print("-" * 30)
+    return exact_ratio, close_ratio
+
+
+def evaluate(
+    prediction: np.ndarray,
+    x: np.ndarray,
+    y: np.ndarray,
+    params: ParameterSet,
+):
+    print("Prediction Shape: {}".format(prediction.shape))
+
+    num: int = x.shape[0]
+    correct: int = 0
+    correct_r: float = 0.0
+    close_r: float = 0.0
+    for i in range(num):
+        should_print = i < 5
+        exact, close = compare(
+            target=y[i],
+            prediction=prediction[i],
+            params=params,
+            print_output=should_print,
+        )
+        if exact == 1.0:
+            correct = correct + 1
+        correct_r += exact
+        close_r += close
+    summary = params.explain()
+    print(
+        "{} Parameters with {} levels (fixed: {})".format(
+            summary["n_variable"], summary["levels"], summary["n_fixed"]
+        )
+    )
+    print(
+        "Got {} out of {} ({:.1f}% perfect); Exact params: {:.1f}%, Close params: {:.1f}%".format(
+            correct,
+            num,
+            correct / num * 100,
+            correct_r / num * 100,
+            close_r / num * 100,
+        )
+    )
+
+
+"""
+Wrap up the whole training process in a standard function. Gets a callback
+to actually make the model, to keep it as flexible as possible.
+# Params:
+# - dataset_name (dataset name)
+# - model_name: (C1..C6,e2e)
+# - model_callback: function taking name,inputs,outputs,data_format and returning a Keras model
+# - epochs: int
+# - dataset_dir: place to find input data
+# - output_dir: place to put outputs
+# - parameters_file (override parameters filename)
+# - dataset_file (override dataset filename)
+# - data_format (channels_first or channels_last)
+# - run_name: to save this run as
+"""
+# LOSS TYPE FOR CUSTOM LOSS FUNCTION
+LOSS_TYPE = "L1"
+SPECTRO_TYPE = "spectro"
+PRINT = 1
+
+# DAWDREAMER EXPORT SETTINGS
+SAMPLE_RATE = 16384
+BUFFER_SIZE = 1024
+SYNTH_PLUGIN = "TAL-NoiseMaker.vst3"
+
+ENGINE = daw.RenderEngine(SAMPLE_RATE, BUFFER_SIZE)
+SYNTH = ENGINE.make_plugin_processor("my_synth", SYNTH_PLUGIN)
+SYNTH.add_midi_note(40, 127, 0, 0.8)
+
+with open("plugin_config/TAL-NoiseMaker-config.json") as f:
+    data = json.load(f)
+
+dico = []
+# Extract the key ID from the JSON data
+key_id = data["parameters"]
+for param in key_id:
+    dico.append(param["id"])
+
+DICO = dico
+
+
+def train_model(
+    # Main options
+    dataset_name: str,
+    model_name: str,
+    epochs: int,
+    model_callback: Callable[[str, int, int, str], keras.Model],
+    dataset_dir: str,
+    output_dir: str,  # Directory names
+    dataset_file: str = None,
+    parameters_file: str = None,
+    run_name: str = None,
+    data_format: str = "channels_last",
+    save_best: bool = True,
+    resume: bool = False,
+    checkpoint: bool = True,
+    model_type: str = "STFT",
+):
+    tf.config.run_functions_eagerly(True)
+    # tf.data.experimental.enable_debug_mode()
+    time_generated = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    if not dataset_file:
+        dataset_file = (
+            os.getcwd() + "/" + dataset_dir + "/" + dataset_name + "_data.hdf5"
+        )
+    if not parameters_file:
+        parameters_file = (
+            os.getcwd() + "/" + dataset_dir + "/" + dataset_name + "_params.pckl"
+        )
+    if not run_name:
+        run_name = dataset_name + "_" + model_name
+
+    model_file = f"{output_dir}/model/{run_name}_{time_generated}"
+    if not os.path.exists(model_file):
+        os.makedirs(model_file)
+    best_model_file = f"{output_dir}/best_checkpoint/{run_name}_best_{time_generated}"
+    if not os.path.exists(best_model_file):
+        os.makedirs(best_model_file)
+    if resume:
+        # checkpoint_model_file = f"{output_dir}/{run_name}_checkpoint_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
+        # history_file = f"{output_dir}/{run_name}_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
+        checkpoint_model_file = (
+            f"{output_dir}/checkpoints/InverSynth_C6XL_checkpoint_20240123-100644"
+        )
+        history_file = f"{output_dir}/history/InverSynth_C6XL_20240123-100644"
+
+    print(tf.config.list_physical_devices("GPU"))
+    gpu_avail = len(tf.config.list_physical_devices("GPU"))  # True/False
+    cuda_gpu_avail = len(tf.config.list_physical_devices("GPU"))  # True/False
+
+    print("+" * 30)
+    print(f"++ {run_name}")
+    print(
+        f"Running model: {model_name} on dataset {dataset_file} (parameters {parameters_file}) for {epochs} epochs"
+    )
+    print(f"Saving model in {output_dir} as {model_file}")
+    print(f"Saving history as {history_file}")
+    print(f"GPU: {gpu_avail}, with CUDA: {cuda_gpu_avail}")
+    print("+" * 30)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Get training and validation generators
+    params = {"data_file": dataset_file, "batch_size": 64, "shuffle": True}
+
+    model: keras.Model = None
+    if resume and os.path.exists(checkpoint_model_file):
+        history = pd.read_csv(history_file)
+        # Note - its zero indexed in the file, but 1 indexed in the display
+        initial_epoch: int = max(history.iloc[:, 0]) + 1
+        # epochs:int = initial_epoch
+        print(
+            f"Resuming from model file: {checkpoint_model_file} after epoch {initial_epoch}"
+        )
+        model = keras.models.load_model(
+            checkpoint_model_file,
+            custom_objects={
+                "top_k_mean_accuracy": top_k_mean_accuracy,
+                "Spectrogram": Spectrogram,
+                "custom_spectral_loss": custom_spectral_loss,
+                "CustomLoss": CustomLoss,
+            },
+        )
+
+        return model, parameters_file
+
+
+def inferrence(model: keras.Model, parameters_file: str, file_path: str, file_id: str):
+    # Start infer
+
+    with open(parameters_file, "rb") as f:
+        parameters: ParameterSet = load(f)
+
+    print("++++" * 5)
+    print("Pushing to trained model")
+    print("++++" * 5)
+
+    Valid = False
+    while Valid == False:
+        namefile = file_path
+        if os.path.exists(namefile):
+            Valid = True
+        else:
+            raise("File Path invalid, try again ")
+
+    try:
+        newpred = model.predict(audio_importer(str(f"{namefile}")))
+    except: 
+        raise "Crashed"
+    
+    predlist: List[ParamValue] = parameters.decode(newpred[0])
+    df = pd.DataFrame(predlist)
+
+    print(df)
+    df = df.drop(["encoding"], axis=1)
+    # saving the dataframe
+
+    print("Outputting CSV config in " + str(f"temp/"))
+
+    csv_path = str(f"temp/{file_id}_config.csv")
+    
+    xml_path_wow = (f"temp/{file_id}_config.noisemakerpreset")
+    df.to_csv(csv_path)
+    
+    xml_path = convert_csv_to_preset(csv_path, xml_path_wow)
+    # export(prediction, X, y, parameters)
+    # Loop through the rows of the DataFrame
+    i = 0
+    for values in df["value"].values:
+        # Set parameters using DataFrame values
+        SYNTH.set_parameter(DICO[i], values)
+        # (MIDI note, velocity, start, duration)
+        i += 1
+    # Setting volume to 0.9
+    SYNTH.set_parameter(1, 0.9)
+    # Set up the processing graph
+    graph = [
+        # synth takes no inputs, so we give an empty list.
+        (SYNTH, []),
+    ]
+
+    ENGINE.load_graph(graph)
+    ENGINE.render(1)
+    data = ENGINE.get_audio()
+    try:
+        data = librosa.to_mono(data).transpose()
+    except:
+        tf.print("ERROR" * 100)
+        df = df.fillna(0)
+        data = df.to_numpy()
+        data = librosa.to_mono(data).transpose()
+        tf.print("crashed, nan in generation")
+        synth_params = dict(SYNTH.get_patch())
+        print(synth_params)
+
+    df = pd.DataFrame(data)
+
+    # penalty=1000000
+    # df = pd.DataFrame(data)
+    # df = df.fillna(0)
+    # data = df.to_numpy()
+
+    output_file_path = str(f"temp/{file_id}_generated.wav")
+
+    wavfile.write(output_file_path, SAMPLE_RATE, data)
+
+    return file_path, xml_path, output_file_path
+
+
+def generate_audio(df_params):
+    # Loop through the rows of the DataFrame
+    i = 0
+    penalty = 1
+    for param in df_params:
+        # Set parameters using DataFrame values
+        SYNTH.set_parameter(DICO[i], param.value)
+        # (MIDI note, velocity, start, duration)
+        i += 1
+    # Set up the processing graph
+    graph = [
+        # synth takes no inputs, so we give an empty list.
+        (SYNTH, []),
+    ]
+
+    ENGINE.load_graph(graph)
+    ENGINE.render(1)
+    data = ENGINE.get_audio()
+    try:
+        data = librosa.to_mono(data).transpose()
+    except:
+        print("ERROR" * 100)
+        df = pd.DataFrame(data)
+        df = df.fillna(0)
+        data = df.to_numpy()
+        data = librosa.to_mono(data).transpose()
+
+    result = np.array(data)
+    return result, penalty

back/models/runner.py

@@ -0,0 +1,82 @@
+import argparse
+
+from models.launch import train_model
+from models.spectrogram_cnn import get_model as get_spectrogram
+
+
+def standard_run_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Setup and train a model, storing the output"
+    )
+    parser.add_argument(
+        "--model",
+        dest="model_name",
+        type=str,
+        choices=["C1", "C2", "C3", "C4", "C5", "C6", "C6XL", "e2e"],
+        default="e2e",
+        help="Model architecture to run",
+    )
+    parser.add_argument(
+        "--dataset_name",
+        default="InverSynth",
+        help='Name of the dataset to use - other filenames are generated from this. If you have a file "modelname_data.hdf5", put in "modelname"',
+    )
+    parser.add_argument(
+        "--epochs", type=int, default=100, help="How many epochs to run"
+    )
+    parser.add_argument(
+        "--dataset_dir",
+        default="test_datasets",
+        help="Directory full of datasets to use",
+    )
+    parser.add_argument(
+        "--output_dir",
+        default="output",
+        help="Directory to store the final model and history",
+    )
+    parser.add_argument(
+        "--dataset_file", default=None, help="Specify an exact dataset file to use"
+    )
+    parser.add_argument(
+        "--parameters_file",
+        default=None,
+        help="Specify an exact parameters file to use",
+    )
+    parser.add_argument(
+        "--data_format",
+        type=str,
+        choices=["channels_last", "channels_first"],
+        default="channels_last",
+        help="Image data format for Keras. If CPU only, has to be channels_last",
+    )
+    parser.add_argument(
+        "--run_name",
+        type=str,
+        dest="run_name",
+        help="Name to save the output under. Defaults to dataset_name + model",
+    )
+    parser.add_argument(
+        "--resume",
+        dest="resume",
+        action="store_const",
+        const=True,
+        default=False,
+        help="Look for a checkpoint file to resume from",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+
+    print("Starting model runner")
+    # Get a standard parser, and the arguments out of it
+    parser = standard_run_parser()
+    args = parser.parse_args()
+    setup = vars(args)
+
+    print("Parsed arguments")
+    # Figure out the model callback
+    model_callback = get_spectrogram
+
+    # Actually train the model
+    train_model(model_callback=model_callback, **setup)

back/models/spectrogram_cnn.py

@@ -0,0 +1,134 @@
+import numpy as np
+
+# import keras
+from kapre.time_frequency import Spectrogram
+from tensorflow import keras
+
+from generators.generator import *
+from models.common.architectures import layers_map
+
+
+"""
+The STFT spectrogram of the input signal is fed
+into a 2D CNN that predicts the synthesizer parameter
+configuration. This configuration is then used to produce
+a sound that is similar to the input sound.
+"""
+
+
+"""Model Architecture"""
+# @ paper:
+# 1 2D Strided Convolution Layer C(38,13,26,13,26)
+# where C(F,K1,K2,S1,S2) stands for a ReLU activated
+# 2D strided convolutional layer with F filters in size of (K1,K2)
+# and strides (S1,S2).
+
+
+def assemble_model(
+    src: np.ndarray,
+    n_outputs: int,
+    arch_layers: list,
+    n_dft: int = 512,  # Orig:128
+    n_hop: int = 256,  #  Orig:64
+    data_format: str = "channels_first",
+) -> keras.Model:
+
+    inputs = keras.Input(shape=src.shape, name="stft")
+
+    # @paper: Spectrogram based CNN that receives the (log) spectrogram matrix as input
+
+    # @kapre:
+    # abs(Spectrogram) in a shape of 2D data, i.e.,
+    # `(None, n_channel, n_freq, n_time)` if `'channels_first'`,
+    # `(None, n_freq, n_time, n_channel)` if `'channels_last'`,
+    x = Spectrogram(
+        n_dft=n_dft,
+        n_hop=n_hop,
+        input_shape=src.shape,
+        trainable_kernel=True,
+        name="static_stft",
+        image_data_format=data_format,
+        return_decibel_spectrogram=True,
+    )(inputs)
+
+    # Swaps order to match the paper?
+    # TODO: dig in to this (GPU only?)
+    if data_format == "channels_first":  # n_channel, n_freq, n_time)
+        x = keras.layers.Permute((1, 3, 2))(x)
+    else:
+        x = keras.layers.Permute((2, 1, 3))(x)
+
+    # x = keras.layers.Conv2D(64,(3,3),strides=(2,2),activation="relu",data_format="channels_last", padding='same')(x)
+    # x = keras.layers.Conv2D(128,(3,3),strides=(2,2),activation="relu",data_format="channels_last", padding='same')(x)
+    # x = keras.layers.Conv2D(128,(3,4),strides=(2,3),activation="relu",data_format="channels_last", padding='same')(x)
+    # x = keras.layers.Conv2D(128,(3,3),strides=(2,2),activation="relu",data_format="channels_last", padding='same')(x)
+    # x = keras.layers.Conv2D(256,(3,3),strides=(2,2),activation="relu",data_format="channels_last", padding='same')(x)
+    # x = keras.layers.Conv2D(256,(3,3),strides=(1,2),activation="relu",data_format="channels_last", padding='same')(x)
+    for arch_layer in arch_layers:
+        x = keras.layers.Conv2D(
+            arch_layer.filters,
+            arch_layer.window_size,
+            strides=arch_layer.strides,
+            activation=arch_layer.activation,
+            data_format=data_format,
+            padding='same'
+        )(x)
+
+    # Flatten down to a single dimension
+    x = keras.layers.Flatten()(x)
+
+    # @paper: sigmoid activations with binary cross entropy loss
+    # @paper: FC-512
+    x = keras.layers.Dense(512)(x)
+
+    # @paper: FC-368(sigmoid)
+    outputs = keras.layers.Dense(n_outputs, activation="sigmoid", name="predictions")(x)
+
+    return keras.Model(inputs=inputs, outputs=outputs)
+
+
+"""
+Standard callback to get a model ready to train
+"""
+
+
+def get_model(
+    model_name: str, inputs: int, outputs: int, data_format: str = "channels_last"
+) -> keras.Model:
+    arch_layers = layers_map.get("C1")
+    if model_name in layers_map:
+        arch_layers = layers_map.get(model_name)
+    else:
+        print(
+            f"Warning: {model_name} is not compatible with the spectrogram model. C1 Architecture will be used instead."
+        )
+    return assemble_model(
+        np.zeros([1, inputs]),
+        n_outputs=outputs,
+        arch_layers=arch_layers,
+        data_format=data_format,
+    )
+
+
+if __name__ == "__main__":
+
+    from models.launch import train_model, inferrence
+    from models.runner import standard_run_parser
+
+    # Get a standard parser, and the arguments out of it
+    parser = standard_run_parser()
+    args = parser.parse_args()
+    setup = vars(args)
+    print(setup)
+    # distinguish model type for reshaping
+    setup["model_type"] = "STFT"
+    # tf.config.run_functions_eagerly(True)
+    # Actually train the model
+    
+    model, parameters_file = train_model(model_callback=get_model, **setup)
+    
+    file_path, csv_path = inferrence(model, parameters_file)
+    
+    print(file_path)
+    
+    print(csv_path)

back/output.xml

@@ -0,0 +1 @@
+<tal curprogram="0" version="1.7" presetName="CH Chordionator III FN" path="Factory Presets/CHORD/CH Chordionator III FN.noisemakerpreset"><programs><program programname="CH Chordionator III FN" unknown="0.5" volume="0" -="0" filtertype="0.63636364" cutoff="0.6666666666666666" resonance="0.9333333333333333" keyfollow="0.26666666666666666" filtercontour="0.06666666666666667" filterattack="0.13333333333333333" filterdecay="0.26666666666666666" filtersustain="0.3333333333333333" filterrelease="0.13333333333333333" ampattack="0.8" ampdecay="1.0" ampsustain="0.4666666666666667" amprelease="0.13333333333333333" osc1volume="0.26666666666666666" osc2volume="0.9333333333333333" osc3volume="0.3333333333333333" oscmastertune="0" osc1tune="0.6" osc2tune="0.8" osc1finetune="0.6666666666666666" osc2finetune="0.2" osc1waveform="0.0" osc2waveform="0.5" oscsync="1.0" lfo1waveform="0.6" lfo2waveform="1.0" lfo1rate="0.6666666666666666" lfo2rate="0.9333333333333333" lfo1amount="0.06666666666666667" lfo2amount="1.0" lfo1destination="0.2857142857142857" lfo2destination="0.14285714285714285" lfo1phase="0.8666666666666667" lfo2phase="0.9333333333333333" osc2fm="0.4" osc2phase="0.4666666666666667" osc1pw="0.7333333333333333" osc1phase="0.5333333333333333" transpose="0" freeadattack="0.3333333333333333" freeaddecay="0.26666666666666666" freeadamount="0.5333333333333333" freeaddestination="0.2" lfo1sync="1.0" lfo1keytrigger="0" lfo2sync="0.0" lfo2keytrigger="0" portamento="0" portamentomode="0" voices="0" velocityvolume="0" velocitycontour="0" velocitycutoff="0" pitchwheelcutoff="0" pitchwheelpitch="0" ringmodulation="0.4666666666666667" chorus1enable="0.0" chorus2enable="1.0" reverbwet="0.4" reverbdecay="0.8666666666666667" reverbpredelay="0.5333333333333333" reverbhighcut="0.4666666666666667" reverblowcut="0.4666666666666667" oscbitcrusher="0.2" highpass="0.06666666666666667" detune="0.13333333333333333" vintagenoise="0.9333333333333333" envelopeeditordest1="0" envelopeeditorspeed="0" envelopeeditoramount="0" envelopeoneshot="0" envelopefixtempo="0" filterdrive="0.0" delaywet="0.13333333333333333" delaytime="1.0" delaysync="1.0" delayfactorl="0.0" delayfactorr="0.0" delayhighshelf="0.4666666666666667" delaylowshelf="1.0" delayfeedback="0.5333333333333333" /></programs><midimap /></tal>

back/plugin_config/TAL-NoiseMaker-config.json

@@ -0,0 +1,422 @@
+{
+    "parameters": [
+        {
+            "id": 11,
+            "name": "Amp Attack",
+            "values": "-"
+        },
+        {
+            "id": 12,
+            "name": "Amp Decay",
+            "values": "-"
+        },
+        {
+            "id": 14,
+            "name": "Amp Release",
+            "values": "-"
+        },
+        {
+            "id": 13,
+            "name": "Amp Sustain",
+            "values": "-"
+        },
+        {
+            "id": 58,
+            "name": "Chorus 1 Enable",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 59,
+            "name": "Chorus 2 Enable",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 3,
+            "name": "Filter Cutoff",
+            "values": "-"
+        },
+        {
+            "id": 81,
+            "name": "Delay x2 L",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 82,
+            "name": "Delay x2 R",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 85,
+            "name": "Delay Feedback",
+            "values": "-"
+        },
+        {
+            "id": 83,
+            "name": "Delay High Shelf",
+            "values": "-"
+        },
+        {
+            "id": 84,
+            "name": "Delay Low Shelf",
+            "values": "-"
+        },
+        {
+            "id": 80,
+            "name": "Delay Sync",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 79,
+            "name": "Delay Time",
+            "values": "-"
+        },
+        {
+            "id": 78,
+            "name": "Delay Wet",
+            "values": "-"
+        },
+        {
+            "id": 67,
+            "name": "Master Detune",
+            "values": "-"
+        },
+        {
+            "id": 7,
+            "name": "Filter Attack",
+            "values": "-"
+        },
+        {
+            "id": 6,
+            "name": "Filter Contour",
+            "values": "-"
+        },
+        {
+            "id": 8,
+            "name": "Filter Decay",
+            "values": "-"
+        },
+        {
+            "id": 77,
+            "name": "Filter Drive",
+            "values": "-"
+        },
+        {
+            "id": 10,
+            "name": "Filter Release",
+            "values": "-"
+        },
+        {
+            "id": 9,
+            "name": "Filter Sustain",
+            "values": "-"
+        },
+        {
+            "id": 2,
+            "name": "Filter Type",
+            "values": [0.0, 0.09090909, 0.18181818, 0.27272727, 0.36363636, 0.45454545, 0.54545455, 0.63636364, 0.72727273, 0.81818182, 0.90909091, 1.0]
+        },
+        {
+            "id": 43,
+            "name": "Free Ad Amount",
+            "values": "-"
+        },
+        {
+            "id": 41,
+            "name": "Free Ad Attack",
+            "values": "-"
+        },
+        {
+            "id": 42,
+            "name": "Free Ad Decay",
+            "values": "-"
+        },
+        {
+            "id": 44,
+            "name": "Free Ad Destination",
+            "values": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
+        },
+        {
+            "id": 66,
+            "name": "Master High Pass",
+            "values": "-"
+        },
+        {
+            "id": 5,
+            "name": "Filter Keyfollow",
+            "values": "-"
+        },
+        {
+            "id": 30,
+            "name": "Lfo 1 Amount",
+            "values": "-"
+        },
+        {
+            "id": 32,
+            "name": "Lfo 1 Destination",
+            "values": [0.0, 0.14285714285714285, 0.2857142857142857, 0.42857142857142855, 0.5714285714285714, 0.7142857142857142, 0.8571428571428571, 1.0]
+        },
+        {
+            "id": 34,
+            "name": "Lfo 1 Phase",
+            "values": "-"
+        },
+        {
+            "id": 28,
+            "name": "Lfo 1 Rate",
+            "values": "-"
+        },
+        {
+            "id": 45,
+            "name": "Lfo 1 Sync",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 26,
+            "name": "Lfo 1 Waveform",
+            "values": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
+        },
+        {
+            "id": 31,
+            "name": "Lfo 2 Amount",
+            "values": "-"
+        },
+        {
+            "id": 33,
+            "name": "Lfo 2 Destination",
+            "values": [0.0, 0.14285714285714285, 0.2857142857142857, 0.42857142857142855, 0.5714285714285714, 0.7142857142857142, 0.8571428571428571, 1.0]
+        },
+        {
+            "id": 35,
+            "name": "Lfo 2 Phase",
+            "values": "-"
+        },
+        {
+            "id": 29,
+            "name": "Lfo 2 Rate",
+            "values": "-"
+        },
+        {
+            "id": 47,
+            "name": "Lfo 2 Sync",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 27,
+            "name": "Lfo 2 Waveform",
+            "values": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
+        },
+        {
+            "id": 21,
+            "name": "Osc 1 Fine Tune",
+            "values": "-"
+        },
+        {
+            "id": 39,
+            "name": "Osc 1 Phase",
+            "values": "-"
+        },
+        {
+            "id": 38,
+            "name": "Osc 1 PW",
+            "values": "-"
+        },
+        {
+            "id": 19,
+            "name": "Osc 1 Tune",
+            "values": "-"
+        },
+        {
+            "id": 15,
+            "name": "Osc 1 Volume",
+            "values": "-"
+        },
+        {
+            "id": 23,
+            "name": "Osc 1 Waveform",
+            "values":[0.0, 0.5, 1.0]
+        },
+        {
+            "id": 22,
+            "name": "Osc 2 Fine Tune",
+            "values": "-"
+        },
+        {
+            "id": 36,
+            "name": "Osc 2 FM",
+            "values": "-"
+        },
+        {
+            "id": 37,
+            "name": "Osc 2 Phase",
+            "values": "-"
+        },
+        {
+            "id": 20,
+            "name": "Osc 2 Tune",
+            "values": "-"
+        },
+        {
+            "id": 16,
+            "name": "Osc 2 Volume",
+            "values": "-"
+        },
+        {
+            "id": 24,
+            "name": "Osc 2 Waveform",
+            "values":[0.0, 0.5, 1.0]
+        },
+        {
+            "id": 17,
+            "name": "Osc 3 Volume",
+            "values": "-"
+        },
+        {
+            "id": 65,
+            "name": "Osc Bitcrusher",
+            "values": "-"
+        },
+        {
+            "id": 25,
+            "name": "Osc Sync",
+            "values": [0.0, 1.0]
+        },
+        {
+            "id": 4,
+            "name": "Filter Resonance",
+            "values": "-"
+        },
+        {
+            "id": 61,
+            "name": "Reverb Decay",
+            "values": "-"
+        },
+        {
+            "id": 63,
+            "name": "Reverb High Cut",
+            "values": "-"
+        },
+        {
+            "id": 64,
+            "name": "Reverb Low Cut",
+            "values": "-"
+        },
+        {
+            "id": 62,
+            "name": "Reverb Pre Delay",
+            "values": "-"
+        },
+        {
+            "id": 60,
+            "name": "Reverb Wet",
+            "values": "-"
+        },
+        {
+            "id": 57,
+            "name": "Ringmodulation",
+            "values": "-"
+        },
+        {
+            "id": 68,
+            "name": "Vintage Noise",
+            "values": "-"
+        }
+      ],
+
+    "fixed_parameters": [
+        {
+            "id": 73,
+            "name": "Envelope Amount",
+            "value": 0.0
+        },
+        {
+            "id": 71,
+            "name": "Envelope Destination",
+            "value": 0.0
+        },
+        {
+            "id": 72,
+            "name": "Envelope Speed",
+            "value": 0.0
+        },
+        {
+            "id": 75,
+            "name": "Envelope Fix Tempo",
+            "value": 0.0
+        },
+        {
+            "id": 74,
+            "name": "Envelope One Shot Mode",
+            "value": 0.0
+        },
+        {
+            "id": 46,
+            "name": "Lfo 1 Keytrigger",
+            "value": 0.0
+        },
+        {
+            "id": 48,
+            "name": "Lfo 2 Keytrigger",
+            "value": 0.0
+        },
+        {
+            "id": 18,
+            "name": "Osc Mastertune",
+            "value": 0.5
+        },
+        {
+            "id": 55,
+            "name": "Pitchwheel Cutoff",
+            "value": 0.0
+        },
+        {
+            "id": 56,
+            "name": "Pitchwheel Pitch",
+            "value": 0.0
+        },
+        {
+            "id": 49,
+            "name": "Portamento Amount",
+            "value": 0.0
+        },
+        {
+            "id": 50,
+            "name": "Portamento Mode",
+            "value": 0.0
+        },
+        {
+            "id": 40,
+            "name": "Transpose",
+            "value": 0.5
+        },
+        {
+            "id": 53,
+            "name": "Velocity Contour",
+            "value": 0.0
+        },
+        {
+            "id": 54,
+            "name": "Velocity Filter",
+            "value": 0.0
+        },
+        {
+            "id": 52,
+            "name": "Velocity Volume",
+            "value": 0.0
+        },
+        {
+            "id": 51,
+            "name": "Voices",
+            "value": 0.0
+        },
+        {
+            "id": 1,
+            "name": "Master Volume",
+            "value": 0.6
+        },
+        {
+            "id": 76,
+            "name": "Envelope Reset",
+            "value": 0.0
+        }
+      ]
+}

back/plugin_config/gen_config_libTAL-NoiseMaker.so.json

@@ -0,0 +1,435 @@
+{
+    "parameters": [
+        {
+            "id": 0,
+            "name": "-",
+            "value": 0.0
+        },
+        {
+            "id": 1,
+            "name": "Master Volume",
+            "value": 0.0
+        },
+        {
+            "id": 2,
+            "name": "Filter Type",
+            "value": 0.0
+        },
+        {
+            "id": 3,
+            "name": "Filter Cutoff",
+            "value": 0.0
+        },
+        {
+            "id": 4,
+            "name": "Filter Resonance",
+            "value": 0.0
+        },
+        {
+            "id": 5,
+            "name": "Filter Keyfollow",
+            "value": 0.0
+        },
+        {
+            "id": 6,
+            "name": "Filter Contour",
+            "value": 0.0
+        },
+        {
+            "id": 7,
+            "name": "Filter Attack",
+            "value": 0.0
+        },
+        {
+            "id": 8,
+            "name": "Filter Decay",
+            "value": 0.0
+        },
+        {
+            "id": 9,
+            "name": "Filter Sustain",
+            "value": 0.0
+        },
+        {
+            "id": 10,
+            "name": "Filter Release",
+            "value": 0.0
+        },
+        {
+            "id": 11,
+            "name": "Amp Attack",
+            "value": 0.0
+        },
+        {
+            "id": 12,
+            "name": "Amp Decay",
+            "value": 0.0
+        },
+        {
+            "id": 13,
+            "name": "Amp Sustain",
+            "value": 0.0
+        },
+        {
+            "id": 14,
+            "name": "Amp Release",
+            "value": 0.0
+        },
+        {
+            "id": 15,
+            "name": "Osc 1 Volume",
+            "value": 0.0
+        },
+        {
+            "id": 16,
+            "name": "Osc 2 Volume",
+            "value": 0.0
+        },
+        {
+            "id": 17,
+            "name": "Osc 3 Volume",
+            "value": 0.0
+        },
+        {
+            "id": 18,
+            "name": "Osc Mastertune",
+            "value": 0.0
+        },
+        {
+            "id": 19,
+            "name": "Osc 1 Tune",
+            "value": 0.0
+        },
+        {
+            "id": 20,
+            "name": "Osc 2 Tune",
+            "value": 0.0
+        },
+        {
+            "id": 21,
+            "name": "Osc 1 Fine Tune",
+            "value": 0.0
+        },
+        {
+            "id": 22,
+            "name": "Osc 2 Fine Tune",
+            "value": 0.0
+        },
+        {
+            "id": 23,
+            "name": "Osc 1 Waveform",
+            "value": 0.0
+        },
+        {
+            "id": 24,
+            "name": "Osc 2 Waveform",
+            "value": 0.0
+        },
+        {
+            "id": 25,
+            "name": "Osc Sync",
+            "value": 0.0
+        },
+        {
+            "id": 26,
+            "name": "Lfo 1 Waveform",
+            "value": 0.0
+        },
+        {
+            "id": 27,
+            "name": "Lfo 2 Waveform",
+            "value": 0.0
+        },
+        {
+            "id": 28,
+            "name": "Lfo 1 Rate",
+            "value": 0.0
+        },
+        {
+            "id": 29,
+            "name": "Lfo 2 Rate",
+            "value": 0.0
+        },
+        {
+            "id": 30,
+            "name": "Lfo 1 Amount",
+            "value": 0.0
+        },
+        {
+            "id": 31,
+            "name": "Lfo 2 Amount",
+            "value": 0.0
+        },
+        {
+            "id": 32,
+            "name": "Lfo 1 Destination",
+            "value": 0.0
+        },
+        {
+            "id": 33,
+            "name": "Lfo 2 Destination",
+            "value": 0.0
+        },
+        {
+            "id": 34,
+            "name": "Lfo 1 Phase",
+            "value": 0.0
+        },
+        {
+            "id": 35,
+            "name": "Lfo 2 Phase",
+            "value": 0.0
+        },
+        {
+            "id": 36,
+            "name": "Osc 2 FM",
+            "value": 0.0
+        },
+        {
+            "id": 37,
+            "name": "Osc 2 Phase",
+            "value": 0.0
+        },
+        {
+            "id": 38,
+            "name": "Osc 1 PW",
+            "value": 0.0
+        },
+        {
+            "id": 39,
+            "name": "Osc 1 Phase",
+            "value": 0.0
+        },
+        {
+            "id": 40,
+            "name": "Transpose",
+            "value": 0.0
+        },
+        {
+            "id": 41,
+            "name": "Free Ad Attack",
+            "value": 0.0
+        },
+        {
+            "id": 42,
+            "name": "Free Ad Decay",
+            "value": 0.0
+        },
+        {
+            "id": 43,
+            "name": "Free Ad Amount",
+            "value": 0.0
+        },
+        {
+            "id": 44,
+            "name": "Free Ad Destination",
+            "value": 0.0
+        },
+        {
+            "id": 45,
+            "name": "Lfo 1 Sync",
+            "value": 0.0
+        },
+        {
+            "id": 46,
+            "name": "Lfo 1 Keytrigger",
+            "value": 0.0
+        },
+        {
+            "id": 47,
+            "name": "Lfo 2 Sync",
+            "value": 0.0
+        },
+        {
+            "id": 48,
+            "name": "Lfo 2 Keytrigger",
+            "value": 0.0
+        },
+        {
+            "id": 49,
+            "name": "Portamento Amount",
+            "value": 0.0
+        },
+        {
+            "id": 50,
+            "name": "Portamento Mode",
+            "value": 0.0
+        },
+        {
+            "id": 51,
+            "name": "Voices",
+            "value": 0.0
+        },
+        {
+            "id": 52,
+            "name": "Velocity Volume",
+            "value": 0.0
+        },
+        {
+            "id": 53,
+            "name": "Velocity Contour",
+            "value": 0.0
+        },
+        {
+            "id": 54,
+            "name": "Velocity Filter",
+            "value": 0.0
+        },
+        {
+            "id": 55,
+            "name": "Pitchwheel Cutoff",
+            "value": 0.0
+        },
+        {
+            "id": 56,
+            "name": "Pitchwheel Pitch",
+            "value": 0.0
+        },
+        {
+            "id": 57,
+            "name": "Ringmodulation",
+            "value": 0.0
+        },
+        {
+            "id": 58,
+            "name": "Chorus 1 Enable",
+            "value": 0.0
+        },
+        {
+            "id": 59,
+            "name": "Chorus 2 Enable",
+            "value": 0.0
+        },
+        {
+            "id": 60,
+            "name": "Reverb Wet",
+            "value": 0.0
+        },
+        {
+            "id": 61,
+            "name": "Reverb Decay",
+            "value": 0.0
+        },
+        {
+            "id": 62,
+            "name": "Reverb Pre Delay",
+            "value": 0.0
+        },
+        {
+            "id": 63,
+            "name": "Reverb High Cut",
+            "value": 0.0
+        },
+        {
+            "id": 64,
+            "name": "Reverb Low Cut",
+            "value": 0.0
+        },
+        {
+            "id": 65,
+            "name": "Osc Bitcrusher",
+            "value": 0.0
+        },
+        {
+            "id": 66,
+            "name": "Master High Pass",
+            "value": 0.0
+        },
+        {
+            "id": 67,
+            "name": "Master Detune",
+            "value": 0.0
+        },
+        {
+            "id": 68,
+            "name": "Vintage Noise",
+            "value": 0.0
+        },
+        {
+            "id": 69,
+            "name": "Panic",
+            "value": 0.0
+        },
+        {
+            "id": 70,
+            "name": "MIDI LEARN",
+            "value": 0.0
+        },
+        {
+            "id": 71,
+            "name": "Envelope Destination",
+            "value": 0.0
+        },
+        {
+            "id": 72,
+            "name": "Envelope Speed",
+            "value": 0.0
+        },
+        {
+            "id": 73,
+            "name": "Envelope Amount",
+            "value": 0.0
+        },
+        {
+            "id": 74,
+            "name": "Envelope One Shot Mode",
+            "value": 0.0
+        },
+        {
+            "id": 75,
+            "name": "Envelope Fix Tempo",
+            "value": 0.0
+        },
+        {
+            "id": 76,
+            "name": "Envelope Reset",
+            "value": 0.0
+        },
+        {
+            "id": 77,
+            "name": "Filter Drive",
+            "value": 0.0
+        },
+        {
+            "id": 78,
+            "name": "Delay Wet",
+            "value": 0.0
+        },
+        {
+            "id": 79,
+            "name": "Delay Time",
+            "value": 0.0
+        },
+        {
+            "id": 80,
+            "name": "Delay Sync",
+            "value": 0.0
+        },
+        {
+            "id": 81,
+            "name": "Delay x2 L",
+            "value": 0.0
+        },
+        {
+            "id": 82,
+            "name": "Delay x2 R",
+            "value": 0.0
+        },
+        {
+            "id": 83,
+            "name": "Delay High Shelf",
+            "value": 0.0
+        },
+        {
+            "id": 84,
+            "name": "Delay Low Shelf",
+            "value": 0.0
+        },
+        {
+            "id": 85,
+            "name": "Delay Feedback",
+            "value": 0.0
+        }
+    ],
+    "fixed_parameters": []
+}

back/requirements.txt

@@ -0,0 +1,28 @@
+wheel
+fastapi
+uvicorn[standard]
+python-multipart
+python-dotenv
+aiofiles
+torch
+torchaudio
+torchmetrics
+torchvision
+dataclasses
+dawdreamer
+matplotlib
+pandas
+samplerate
+tensorboard
+tensorflow-estimator
+tensorflow
+scikit-learn
+scipy
+numpy
+numba
+kapre==0.1.7
+keras-applications
+keras-preprocessing
+keras
+librosa
+h5py

back/utils/export_to_excel.py

@@ -0,0 +1,4 @@
+import pandas as pd
+
+file = pd.read_csv(f"output/InverSynth_C6XL_20231201-103344") #Read the dataframe
+file.to_excel(f'foo.xlsx', index=False) #Save the dataframe

back/utils/import csv.py

@@ -0,0 +1,16 @@
+import csv
+
+# Open the CSV file
+with open('InverSynth_00006.wav.csv', 'r') as csvfile:
+    reader = csv.reader(csvfile)
+    
+    # Skip the header row
+    next(reader)
+    
+    # Loop through the rows
+    for row in reader:
+        # Get the floating-point number from the third column
+        value = float(row[2])
+        
+        # Do something with the value
+        print(f"The value is {value}")

back/utils/import json.py

@@ -0,0 +1,13 @@
+import json
+
+# Load the JSON data from a file
+with open('plugin_config/TAL-NoiseMaker-config.json') as f:
+    data = json.load(f)
+
+dico=[]
+# Extract the key ID from the JSON data
+key_id = data['parameters']
+for param in key_id:
+    dico.append(param['id'])
+
+print(dico)

back/utils/synth.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import contextlib
+import glob
+import io
+import json
+import logging
+import multiprocessing
+import os
+import random
+import time
+import traceback
+import sys
+
+from pydub import AudioSegment, silence
+from anyio import Path
+import dawdreamer as daw
+import numpy as np
+from scipy.io import wavfile
+from pydub import AudioSegment
+from pydub.silence import split_on_silence
+import tqdm
+import csv
+
+
+SAMPLE_RATE = 44100
+# Parameters will undergo automation at this buffer/block size.
+BUFFER_SIZE = 128
+PPQN = 960  # Pulses per quarter note.
+
+SYNTH_PLUGIN = "libTAL-NoiseMaker.so"
+# SYNTH_PLUGIN = "C:/Program Files/Common Files/VST3/Surge Synth Team/Surge XT.vst3/Contents/x86_64-win/Surge XT.vst3"
+
+
+def make_sine(freq: float, duration: float, sr=SAMPLE_RATE):
+    """Return sine wave based on freq in Hz and duration in seconds"""
+    N = int(duration * sr)  # Number of samples
+    return np.sin(np.pi*2.*freq*np.arange(N)/sr)
+
+
+
+def gen():
+    # print(f'Current gen: {self.current_num}')
+    engine = daw.RenderEngine(SAMPLE_RATE, BUFFER_SIZE)
+    output_dir = Path("yay")
+
+    # Make a processor and give it the unique name "my_synth", which we use later.
+    synth = engine.make_plugin_processor("my_synth", SYNTH_PLUGIN)
+    assert synth.get_name() == "my_synth"
+    
+    #Outputs the properties names and dumps them in a json
+
+    params = {}
+    for param in range(1,86):
+        params[param] = (synth.get_parameter_name(param))
+    with open(str(f'params_.json'), 'w') as f:
+        json.dump(params, f)
+
+
+    #synth.load_preset("C:/Users/yderre/AppData/Roaming/ToguAudioLine/TAL-NoiseMaker/presets/Factory Presets/DRUM/DR 8bit  Kick II FN.noisemakerpreset")
+
+    # Get the parameters description from the plugin
+    parameters = synth.get_parameters_description()
+    
+    array = []
+    # Create a dictionary with parameter names as keys and their indices as values
+    synth.add_midi_note(40, 127, 0, 0.2)
+
+    # for i in range(0,120):
+    #     print(f"{parameters[i]['name']}")
+    # return
+
+    with open('InverSynth_01998.wav.csv', 'r') as csvfile:
+        reader = csv.reader(csvfile)
+        
+        # Skip the header row
+        next(reader)
+        i=0
+        # Loop through the rows
+        for row in reader:
+            
+            # Get the floating-point number from the third column
+            value = float(row[2])
+            params = {}
+            # (MIDI note, velocity, start, duration)
+            print(f"{parameters[i]['name']} changed from  {parameters[i]['defaultValue']} to {value} ")
+            synth.set_parameter(i, value)
+            i+=1
+            # don't do reverb
+            graph = [
+                # synth takes no inputs, so we give an empty list.
+                (synth, []),
+            ]
+            
+    
+
+    engine.load_graph(graph)
+    engine.render(1)
+    output = engine.get_audio()
+    wavfile.write(
+        str(output_dir/f'test_.wav'), SAMPLE_RATE, output.transpose())
+    synth.open_editor()  # Open the editor, make changes, and clos
+
+
+
+
+if __name__ == "__main__":
+    gen()