mhubert-base

• the checkpoint converted from textless s2st real data

usage:

asrp==0.0.35 # extracted from fairseq repo

# https://huggingface.co/voidful/mhubert-base/resolve/main/mhubert_base_vp_en_es_fr_it3_L11_km1000.bin
# https://keithito.com/LJ-Speech-Dataset/LJ037-0171.wav

import asrp
hc = asrp.HubertCode("voidful/mhubert-base", './mhubert_base_vp_en_es_fr_it3_L11_km1000.bin', 11)
code = hc('./LJ037-0171.wav')['code']

result:

array([991, 393, 946, 215, 215, 327, 487, 487, 219, 219, 522, 522, 975,
       975, 975, 975, 668, 576, 576, 384, 761, 907, 430, 748,  12,  12,
       977, 877, 179, 961, 428, 428, 822,  89, 194, 194, 664, 817, 817,
       146, 146, 146, 283, 283, 352, 352, 428, 428, 812, 523, 143, 105,
       105, 244, 244, 583, 583, 576, 384, 879,  32, 170, 683, 731, 600,
       600, 702,  15,  59, 754, 872, 324, 789, 789, 402, 908, 380, 211,
       179, 961, 207, 950, 321, 113, 327, 327, 932, 148, 148, 202, 393,
       946, 215, 215, 406, 406, 423, 423,   6, 384, 879, 879, 219, 219,
       522, 522, 589, 589, 337, 126, 126, 126, 323, 740, 663, 663, 969,
       969, 969, 506, 506, 506, 545, 545,  85,  85, 297, 297, 265, 675,
       237, 237, 307, 407, 407, 499, 407, 334, 334, 334, 111, 666, 666,
       277, 128, 665, 644, 644, 389, 771,  46,  46, 179, 961, 931, 428,
       822, 822,  89, 194, 194, 664, 765, 765, 302, 302, 205, 205, 521,
       521,  29,  29, 537, 393, 393, 946, 734, 263,  45, 914, 445, 469,
       469, 469, 482, 972, 972, 972, 972, 333, 333, 817, 817, 817, 146,
       146, 146, 283,  88, 352, 352, 915, 143,  79,  79, 868, 868, 220,
       220, 870,  45, 272, 313, 313, 367, 367, 729, 729, 409, 409, 409,
        45, 468, 468, 468, 468, 468, 468, 468, 468, 340, 340, 340, 340,
       340, 340, 340, 340, 380, 660, 555, 555, 208, 417, 942, 605, 193,
       121, 407, 704, 704, 704, 704, 334, 499, 226, 226, 621, 128, 665,
       665, 991, 991, 459, 459, 459, 173, 945, 945, 945, 233, 233, 479,
       479, 479, 479, 330, 776, 776, 655, 655, 655, 837, 837,  81,  81,
       664, 429, 148, 431, 431, 531, 531, 531, 531, 531, 668, 167, 104,
       104, 104,  70,  70, 185, 686,  85,  85,  85, 297, 243, 243, 172,
       172, 871, 877,  89, 194, 664, 470, 470, 152, 152, 152, 429, 429,
       429, 429, 290, 943, 943, 943, 484, 488, 620, 352, 915, 143,  38,
       479, 479, 479, 479, 330, 330, 776, 167, 655, 655, 655, 837, 837,
        81,  81,  81, 284, 284, 377, 377, 663, 969, 969, 969, 555, 555,
       208, 433, 755, 942, 942, 605, 193, 121, 121, 121, 704, 704, 334])

Eval

# https://dl.fbaipublicfiles.com/fairseq/speech_to_speech/vocoder/code_hifigan/mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj/g_00500000

import asrp

hc = Code2Speech('./g_00500000', vocoder='hifigan', end_tok=999, code_begin_pad=0)

# play on notebook
import IPython.display as ipd
ipd.Audio(data=hc(code), autoplay=False, rate=16000)