notebook update

README.md

@@ -14,10 +14,16 @@ This model card provides an easy-to-use API for a *pretrained DAC* [1] whose bac
 
 # Usage
 
+### Load
+```python
+from transformers import AutoModel
+model = AutoModel.from_pretrained('hance-ai/descript-audio-codec', trust_remote_code=True)
+```
 
 
 <!-- 
 - different models for different khz  
+- how to adjust model parameters
 -->
 
 

save_model.ipynb

@@ -101,52 +101,307 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "A new version of the following files was downloaded from https://huggingface.co/hance-ai/descript-audio-codec:\n",
-      "- model.py\n",
-      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
-      "C:\\Users\\dslee\\AppData\\Roaming\\Python\\Python38\\site-packages\\audiotools\\ml\\layers\\base.py:172: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
-      "  model_dict = torch.load(location, \"cpu\")\n",
-      "c:\\Users\\dslee\\anaconda3\\envs\\sound_effect_variation_generation\\lib\\site-packages\\torch\\nn\\utils\\weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.\n",
-      "  WeightNorm.apply(module, name, dim)\n"
-     ]
+     "data": {
+      "text/plain": [
+       "DAC(\n",
+       "  (dac): DAC(\n",
+       "    (encoder): Encoder(\n",
+       "      (block): Sequential(\n",
+       "        (0): Conv1d(1, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "        (1): EncoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (1): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): Snake1d()\n",
+       "            (4): Conv1d(64, 128, kernel_size=(4,), stride=(2,), padding=(1,))\n",
+       "          )\n",
+       "        )\n",
+       "        (2): EncoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(128, 128, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (1): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(128, 128, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(128, 128, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): Snake1d()\n",
+       "            (4): Conv1d(128, 256, kernel_size=(8,), stride=(4,), padding=(2,))\n",
+       "          )\n",
+       "        )\n",
+       "        (3): EncoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(256, 256, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (1): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(256, 256, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(256, 256, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): Snake1d()\n",
+       "            (4): Conv1d(256, 512, kernel_size=(16,), stride=(8,), padding=(4,))\n",
+       "          )\n",
+       "        )\n",
+       "        (4): EncoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(512, 512, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(512, 512, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (1): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(512, 512, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(512, 512, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(512, 512, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(512, 512, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): Snake1d()\n",
+       "            (4): Conv1d(512, 1024, kernel_size=(16,), stride=(8,), padding=(4,))\n",
+       "          )\n",
+       "        )\n",
+       "        (5): Snake1d()\n",
+       "        (6): Conv1d(1024, 1024, kernel_size=(3,), stride=(1,), padding=(1,))\n",
+       "      )\n",
+       "    )\n",
+       "    (quantizer): ResidualVectorQuantize(\n",
+       "      (quantizers): ModuleList(\n",
+       "        (0-8): 9 x VectorQuantize(\n",
+       "          (in_proj): Conv1d(1024, 8, kernel_size=(1,), stride=(1,))\n",
+       "          (out_proj): Conv1d(8, 1024, kernel_size=(1,), stride=(1,))\n",
+       "          (codebook): Embedding(1024, 8)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (decoder): Decoder(\n",
+       "      (model): Sequential(\n",
+       "        (0): Conv1d(1024, 1536, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "        (1): DecoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): Snake1d()\n",
+       "            (1): ConvTranspose1d(1536, 768, kernel_size=(16,), stride=(8,), padding=(4,))\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(768, 768, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(768, 768, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (4): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(768, 768, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "          )\n",
+       "        )\n",
+       "        (2): DecoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): Snake1d()\n",
+       "            (1): ConvTranspose1d(768, 384, kernel_size=(16,), stride=(8,), padding=(4,))\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(384, 384, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(384, 384, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(384, 384, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(384, 384, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (4): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(384, 384, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(384, 384, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "          )\n",
+       "        )\n",
+       "        (3): DecoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): Snake1d()\n",
+       "            (1): ConvTranspose1d(384, 192, kernel_size=(8,), stride=(4,), padding=(2,))\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(192, 192, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(192, 192, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (4): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(192, 192, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "          )\n",
+       "        )\n",
+       "        (4): DecoderBlock(\n",
+       "          (block): Sequential(\n",
+       "            (0): Snake1d()\n",
+       "            (1): ConvTranspose1d(192, 96, kernel_size=(4,), stride=(2,), padding=(1,))\n",
+       "            (2): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(96, 96, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(96, 96, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (3): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(96, 96, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(96, 96, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "            (4): ResidualUnit(\n",
+       "              (block): Sequential(\n",
+       "                (0): Snake1d()\n",
+       "                (1): Conv1d(96, 96, kernel_size=(7,), stride=(1,), padding=(27,), dilation=(9,))\n",
+       "                (2): Snake1d()\n",
+       "                (3): Conv1d(96, 96, kernel_size=(1,), stride=(1,))\n",
+       "              )\n",
+       "            )\n",
+       "          )\n",
+       "        )\n",
+       "        (5): Snake1d()\n",
+       "        (6): Conv1d(96, 1, kernel_size=(7,), stride=(1,), padding=(3,))\n",
+       "        (7): Tanh()\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       ")"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "# load the uploaded model\n",
     "from transformers import AutoModel\n",
-    "model = AutoModel.from_pretrained('hance-ai/descript-audio-codec', trust_remote_code=True)"
+    "model = AutoModel.from_pretrained('hance-ai/descript-audio-codec', \n",
+    "                                  trust_remote_code=True)\n",
+    "model.to('cpu')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "device(type='cpu')"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "model.device"
+    "model."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -168,6 +423,25 @@
     "print('zq.shape:', zq.shape)\n",
     "print('s.shape:', s.shape)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "waveform.shape: torch.Size([1, 1, 441344])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# decoding (from zq -- discrete latent vectors)\n",
+    "waveform = model.decode(zq=zq)\n",
+    "print('waveform.shape:', waveform.shape)"
+   ]
   }
  ],
  "metadata": {

test_DAC.ipynb

@@ -9,41 +9,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 12,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\dslee\\AppData\\Roaming\\Python\\Python38\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "from pathlib import Path\n",
     "\n",
-    "import torch\n",
-    "\n",
     "from model import DAC, DACConfig"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
     "# settings\n",
     "fname = str(Path(os.getcwd()).joinpath('.sample_sound', 'jazz_swing.wav'))\n",
-    "device = 'cpu'"
+    "device = 'cpu'\n",
+    "model_type_by_sampling_freq = '44khz'"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -59,13 +49,13 @@
    ],
    "source": [
     "# load the model\n",
-    "config = DACConfig()\n",
+    "config = DACConfig(model_type_by_sampling_freq=model_type_by_sampling_freq)\n",
     "dac = DAC(config).to(device)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {