diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b7b3c0b41246c9bb69690221776dc294325c7551 --- /dev/null +++ b/.gitignore @@ -0,0 +1,183 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.lh +.history +.vscode +.idea +.trunk +lightning_logs + +# Ignore generated sounds +results/* + +# Mock datasets +datasets_cache/**/* + +notebooks/**/*.wav +notebooks/**/*.mp3 +notebooks/**/*.pth + +# Mock audio +mocks/wav2vec_aligner/audio + +# Ignore flagged from gradio +flagged \ No newline at end of file diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/config/best_speakers_list.py b/config/best_speakers_list.py new file mode 100644 index 0000000000000000000000000000000000000000..54095136bdf5ee4f4de93e3cd644bf1c5b7fc0e0 --- /dev/null +++ b/config/best_speakers_list.py @@ -0,0 +1,968 @@ +# chunk_1 +speaker_ids1 = [ + 32, # US, nan M train-clean-100, min intonation, calm, timbre + 130, # BR, Peter of Buckinghamshire England M train-other-500, timbre, min intonation, calm, slow + 164, # US, Ed Good M train-other-500, timbre, intonation, warm and calm, good voice + 176, # US, Micah Sheppard M train-clean-360, intonation, calm + 68, # US, Alex Buie M train-clean-100, min intonation, energized + 15, # US, Chip M train-other-500, intonation, interesting voice + 145, # US, Mr. Baby Man M train-clean-360, timbre, intonation, interesting voice + 196, # US, Mike Kauffmann M train-clean-360, min intonation, average calm + 160, # BR, deadwhitemales M train-clean-100, intonation, energized, interesting voice + 108, # US, Kevin McAsh M train-clean-360, min intonation, energized + 99, # US, Stewart Wills M train-clean-100, min intonation, avg energy + 50, # US, Dan Threetrees M train-clean-360, min intonation, avg energy + 76, # US, ML Cohen M train-other-500, intonation, energy, timbre + 142, # US, Michael Sirois M train-other-500, intonation, energy + 95, # US, Vinny Bove M train-clean-360, intonation, energy, timbre + 169, # US, Richard Grove M train-other-500, intonation, energy + 147, # US, Eileen George F train-clean-360, intonation, calm + 92, # BR vlooi F train-other-500 teenager voice, intonation, energized + 117, # US, Caitlin Kelly F train-clean-360, intonation, energized, interesting voice + 89, # US, Paula Berinstein F train-clean-360, min intonation, energized + 182, # US Katy Preston F train-other-500, intonation, energized + 105, # US Marian Brown F train-clean-360, min intonation, energized, timbre, quality GOOD + 11, # BR Linton F train-other-500, min intonation, calm + 18, # US Sherry Crowther F train-clean-100, intonation, energized, timbre + 38, # US Kurt Copeland M train-clean-360, intonation, energized + 52, # US, Cori Samuel F train-other-500, intonation, energized, timbre + 21, # US, Kelly Bescherer F train-other-500, intonation, energized, timbre + 48, # US, Rosalind Wills F train-clean-100, intonation, timbre, poor quality, but interesting voice + 63, # US, Linda Wilcox F train-other-500, very poor quality, but VERY interesting intonation + 45, # US, Catharine Eastman F train-clean-100, very quality, interesting intonation + 193, # US, Nomenphile F train-other-500, avg quality, intonation + 207, # US Sage Tyrtle F train-other-500, avg quality, interesting intonation and timbre + 73, # US Claire Goget F train-clean-100, avg quality, intonation + 42, # US Jennifer Crispin F train-clean-360, quality, calm + 113, # US Alice Elizabeth Still F train-other-500, avg quality, intonation + 49, # US Kristen McQuillin F train-clean-100, avg quality, interesting intonation and timbre + 185, # US Kim Braun F train-clean-360, avg quality, avg intonation, but interesting timbre + 165, # US Elisabeth Shields F train-clean-100, avg quality, interesting intonation, timbre + 36, # US chriss the girl F train-other-500, interesting intonation, timbre + 96, # US Kymm Zuckert F train-other-500, avg quality, interesting intonation, timbre of middle age wooman + 102, # US Maureen S. O'Brien F train-clean-100, avg quality, interesting intonation + 64, # US Christiane Levesque F train-clean-360, young voice, calm + 30, # BR, Ophelia Darcy F train-other-500, avg quality, intonation + 0, # US Kristin LeMoine F train-clean-360, avg quality, intonation, fast-speaker + 192, # US, Nocturna F train-clean-100, avg quality, intonation + 146, # US Jim Cadwell M train-other-500, avg quality, intonation + 80, # US Fox in the Stars F train-clean-100, avg quality, intonation + 162, # BR Mike Gardom M train-other-500, avg quality, intonation, timbre + 66, # US Maddie F train-clean-360, intonation, timbre + 124, # US Steve Karafit M train-clean-100, poor quality, intonation, timbre, COOL + 123, # US Sean McGaughey M train-clean-360, poor quality, intonation, timbre + 55, # US, Patricia Oakley F train-clean-360, intonation, timbre + 91, # BR Chris Goringe M train-other-500, intonation, timbre + 171, # US Paul Harvey M train-clean-360, avg quality, intonation, timbre + 61, # US John Greenman M train-other-500, intonation, timbre + 122, # US carnright M train-clean-100, avg intonation, timbre + 82, # BR Andy Minter M train-other-500, intonation, GOOD timbre, + 118, # US Brenda Dayne F train-clean-360, timbre, calm + 106, # US Mark F. Smith M train-clean-360, GOOD timbre, interesting intonation + 187, # US Lenny Glionna Jr. M train-other-500, intonation, timbre + 158, # US Randy Phillips M train-clean-100, avg intonation, timbre + 83, # US Graham Williams M train-other-500, intonation, timbre + 181, # BR Jon Ingram M train-other-500, intonation, timbre + 8, # US Denny Sayers M train-clean-100, timbre + 205, # US Robert Garrison M train-clean-360, timbre, interesting intonation + 172, # US Harvey Chinn M train-other-500, great timbre, intonation +] +# chunk_2 +speaker_ids2 = [ + 305, # US Michael Crowl M train-clean-360, timbre, intonation + 417, # US Kiki Baessell F train-clean-360, timbre, intonation + 241, # US Jean O'Sullivan F train-clean-360, timbre, intonation + 412, # US Nichole Karl F train-other-500, timbre + 294, # US Diana Kiesners F train-clean-360, timbre, intonation + 414, # US Christabel F train-clean-100, great and calm timbre + 251, # US Aaron Benedict M train-other-500, timbre, intonation + 218, # US Joy Chan F train-other-500, timbre, intonation, quality + 259, # BR miette F train-other-500, timbre, intonatio + 275, # US Zale Schafer (Rose May Chamberlin Memorial Foundat F train-clean-360, timbre, intonation + 368, # US Susie G. F train-other-500, teenage voice, timbre, intonation + 327, # BR mjd-s F train-other-500, timbre, intonation + 366, # US pattymarie F train-clean-360, calm, timbre + 399, # US entada F train-clean-360, timbre, intonation + 273, # BR Andrew Lebrun M train-other-500, timbre, intonation + 352, # US Mur Lafferty F train-clean-360, timbre, enery, young voice + 283, # US Bethany Simpson F train-clean-360, timbre, intonation + 248, # BR fieldsofgold M train-other-500, timbre, intonation + 372, # US Jim Mullins M train-clean-360, timbre, intonation, quality + 250, # US Quentin M train-clean-360, unique timbre, intonation, quality + 403, # US Igor Teaforay F train-clean-360, timbre, avg intonation + 375, # US iscatel M train-clean-360, unique timbre, avg intonation + 355, # US Lana Taylor F train-clean-100, unique timbre, intonation + 220, # US Tina Tilney F train-clean-360, unique timbre, high intonation + 392, # US Brooks Jensen M train-clean-360, timbre, avg speed, avg intonation + 255, # US Caroline Mercier F train-clean-360, timbre, avg speed, intonation + 292, # US Tamara R. Schwartz F train-clean-100, timbre, avg speed, intonation + 258, # US Russ Maxwell M train-clean-360, timbre, intonation, unique voice + 233, # US mikenkat M train-clean-360, timbre, intonation + 394, # US swroot F train-clean-360, timbre, intonation, unique voice, good quality + 382, # US Kelli Robinson F train-clean-360, timbre, intonation, avg quality + 322, # US Mitchell Dwyer M train-clean-360, timbre, intonation, avg quality, unique voice + 285, # US koijmonop M train-clean-360, timbre, intonation, avg quality, unique voice + 290, # US J. M. Smallheer F train-clean-360, timbre, intonation, avg quality + 383, # US Mike Roop M train-other-500, timbre, intonation, quality, unique voice + 263, # US Eric S. Piotrowski M train-clean-360, timbre, intonation, avg quality + 373, # BR Brooks Seveer M train-clean-360, timbre, intonation, avg quality, unique voice + 340, # US Nick Gallant M train-clean-100, unique timbre + 331, # BR Luke Venediger M train-other-500, timbre, intonation + 234, # BR Menno M train-other-500, timbre, intonation, unique voice + 398, # BR Sam Fold M train-other-500, timbre, intonation + 254, # BR Rebecca Dittman M train-other-500, timbre, intonation + 264, # US KentF M train-clean-360, timbre, intonation + 214, # US Scott Splavec M train-clean-100, timbre, intonation + 359, # BR Greg Bryant M train-clean-100, timbre, intonation + 253, # US Frank M train-other-500, timbre, intonation + 249, # US Bill Stackpole M train-clean-360, timbre, intonation + 236, # US Matthew Shepherd M train-clean-360, timbre, intonation, unique voice + 390, # US JimmyLogan M train-clean-360, avg intonation, timbre + 216, # US Dave Ranson M train-clean-100, unique voice, timbre, intonation + 215, # US Mark Bradford M train-clean-360, timbre, intonation + 212, # US Glen Hallstrom M train-other-500, timbre, intonation + 314, # US Carl Vonnoh, III M train-clean-360, unique voice, timbre, intonation + 347, # US Anadaxis_Canejia M train-other-500, unique voice, timbre, intonation, good quality + 330, # US Aaron Andradne M train-clean-360, timbre, intonation + 393, # US Tim Lundeen M train-clean-360, timbre, intonation + 40, # BR Justin Brett M train-other-500, unique voice, timbre, intonation, good quality + 386, # US Michael Kirkpatrick M train-clean-360, unique intonation, timbre + 315, # US Jean Crevier M train-other-500, unique intonation, timbre + 349, # US brenthumphries M train-other-500, unique timbre, min intonation + 242, # US J. Hall M train-other-500, unique timbre, intonation + 308, # US Eric Connover M train-other-500, timbre, intonation + 313, # BR Tim Bulkeley M train-other-500, timbre, intonation + 334, # US mawrtea M train-clean-360, unique voice, timbre, intonation +] +# chunk_3 +speaker_ids3 = [ + 469, # US Christian Pecaut M train-clean-360, clear voice, min intonation + 601, # BR Jonathan Horniblow M train-other-500, clear voice, min intonation + 482, # US Estragon M train-clean-360, clear voice, min intonation + 580, # US Kyle M. M train-clean-360, clear voice, min intonation, timbre + 463, # US Chris Hughes M train-other-500, clear voice, min intonation, timbre, energized + 472, # BR Tim Makarios M train-other-500, min intonation + 536, # US Robert Flach M train-other-500, intonation, timbre, unique voice + 569, # US Arouet M train-clean-360, avg intonation, timbre + 523, # US Michael Loftus M train-clean-360, avg intonation, timbre + 454, # US Tim Gregory M train-clean-100, avg intonation + 425, # US RedToby M train-clean-360, avg intonation, timbre, unique voice + 574, # US Daniel Shorten M train-clean-100, avg intonation, timbre, clear voice + 465, # US Leonie Rose F train-clean-100, avg intonation, timbre + 481, # US Scott Sherris M train-clean-360, intonation, timbre + 531, # US Fr. Richard Zeile of Detroit M train-other-500, intonation, timbre, unique voice + 628, # US Bryan Ness M train-clean-100, intonation, timbre, unique voice + 427, # US John Lieder M train-clean-360, intonation, timbre + 527, # US Jason Isbell M or F train-clean-360, intonation, timbre + 468, # US Jenilee F train-other-500, energized, young voice, intonation + 441, # US roolynninms F train-clean-100, energized, intonation + 430, # BR Millbeach F train-other-500, intonation, timbre + 498, # US Chris Gladis M train-clean-100, intonation, timbre + 435, # US Debra Lynn F train-other-500, intonation, timbre + 499, # US Tammy Sanders F train-clean-100, intonation, timbre, unique voice + 429, # US Giles Baker M train-other-500, timbre + 495, # US Janet Friday F train-clean-360, INTONATION, timbre + 456, # US Catherine Fitz F train-clean-360, timbre + 617, # US PJ F train-other-500, timbre + 445, # US Jennette Selig F train-clean-360, intonation, timbre + 552, # US Mim Ritty F train-clean-100, intonation, timbre + 561, # US Lorelle Anderson F train-clean-100, intonation, timbre, quality + 447, # US Lee Ann Howlett F train-clean-360, intonation, timbre, quality + 604, # BR Anne-Marie F train-other-500, intonation, timbre + 450, # US Heather Duncan F train-clean-360, intonation, unique voice + 502, # US Lee Elliott F train-other-500, intonation + 517, # US Madame Tusk F train-other-500, intonation + 605, # BR Classicsfan F train-other-500, intonation, unique voice, timbre + 426, # US Megan Stemm-Wade F train-clean-100, clear voice, min intonation + 544, # US Miranda Stinson F train-clean-360, unique voice + 568, # US Michael Yourshaw M train-other-500, clear voice + 588, # US Kalynda F train-clean-360, clear voice, timbre + 477, # US Patti Brugman F train-other-500, clear voice + 607, # BR Steph F train-other-500, clear voice + 479, # US Wina Hathaway F train-other-500, intonation, timbre + 618, # US Linnea F train-other-500, timbre + 549, # US AmyAG F train-other-500, teenager voice + 596, # US Jo F train-other-500, timbre, min intonation +] +# chunk 4 +speaker_ids4 = [ + 770, # US Pete Williams, Pittsburgh, PA M train-clean-360, clear, calm voice + 730, # US Nick Gisburne M train-other-500, clear, calm voice + 680, # US Kevin Kivikko M train-clean-360, min intonation, calm voice, timbre + 791, # US David Kleparek M train-clean-100, clear, calm voice + 800, # US Jack Farrell M train-clean-360, clear, calm voice, good low tone + 839, # BR Ben Dutton M train-other-500, clear, calm voice, min intonation + 661, # US James Gladwin M train-other-500, calm voice, min intonation + 648, # US Muhammad Mussnoon M train-other-500, timbre + 798, # US Wyatt M train-other-50, calm voice, timbre, avg intonation + 699, # US Michael Macedonia M train-clean-360, calm voice, timbre + 757, # US Roger Melin M train-clean-360, timbre, avg intonation, unique voice + 782, # US Alec Daitsman M train-clean-360, timbre, avg intonation + 816, # US Michael Bradford M train-clean-360, timbre, avg intonation, teenager voice + 846, # US Gayland Darnell M train-clean-360, senior voice, timbre, avg intonation + 831, # US David Federman M train-other-500, timbre, avg intonation + 817, # US texttalker M train-clean-360, timbre, avg intonation + 819, # US n8evv M train-clean-360, timbre, intonation + 672, # BR Stuart Bell M train-other-500, timbre, intonation, clear voice + 837, # US johnb M train-other-500, timbre, intonation + 775, # US Ralph Volpi M train-clean-360, senior voice, timbre, intonation + 685, # US Nikki Sullivan F train-clean-100, clear voice, avg intonation, timbre + 666, # US Kelly Dougherty M train-clean-360, clear voice, avg intonation, timbre + 809, # US Anna-Maria Viola F train-clean-360, timbre, avg intonation + 640, # US David A. Stokely M train-other-500, emotional, timbre, intonation, unique voice + 818, # US Matt Warzel F train-clean-360, timbre, intonation + 711, # US Julie Bynum F train-clean-360, senior voice, timbre, min intonation + 766, # US Clive Catterall M train-other-500, avg intonation, timbre + 649, # US Scarlett! F train-clean-360, intonation, timbre + 732, # US Tysto M/F train-clean-360, intonation, timbre + 769, # US, Alan Brown M train-other-500, avg intonation, timbre + 677, # US Allyson Hester F train-other-500, intonation, timbre, unique voice + 833, # US Alana Jordan F train-clean-360, intonation, timbre + 676, # US Jennifer F train-clean-100, intonation, timbre + 710, # US Sheila Morton F train-clean-100, intonation, timbre, unique voice + 645, # US Micah F train-other-500, intonation, timbre, unique voice, teenager voice + 771, # US Isosceles F train-clean-360, intonation, timbre + 636, # US Matthew Howell F train-other-500, timbre, avg intonation + 822, # US kristiface F train-clean-360, timbre, intonation, unique voice + 830, # US musici123 F train-other-500, timbre, intonation, unique voice + 657, # US Shannon F train-other-500, timbre, intonation + 797, # US Chris Jones F train-other-500, timbre, intonation, calm voice + 835, # BR Rachel Lintern F train-other-500, timbre, intonation + 761, # US Susan Umpleby F train-clean-100, timbre, intonation, unique voice + 653, # US cucciasv F train-other-500, timbre, avg intonation + 751, # US Ralph Snelson M train-other-500, timbre, avg intonation, unique voice + 848, # BR senshisteph F train-other-500, timbre, intonation + 808, # US M. J. Boyle F train-other-500, timbre, intonation, senior voice + 840, # US B. Grebe F train-clean-360, timbre, intonation, unique voice + 742, # US Katherine Holt F train-other-500, timbre, intonation + 834, # US Serin F train-other-500, unique voice + 668, # US Jan Baxter F train-clean-360, intoantion, timbre, calm voice + 752, # US Cat Schirf F train-other-500, intoantion, timbre + 641, # US Eliza Horne F train-other-500, intoantion, timbre + 644, # US Cynthia Zocca F train-clean-360, intoantion, timbre + 781, # US Megan Kunkel F train-other-500, intoantion, timbre, teenager voice + 727, # US Jodi Krangle F train-clean-360, timbre + 719, # US Charlene V. Smith F train-other-500, intoantion, timbre + 804, # BR FirstKnight F train-other-500, unique voice, intoantion, timbre + 675, # US inkwelldragon F train-clean-360, intoantion, timbre + 697, # US Jennie Hughes F train-other-500, timbre + 731, # IND Priya, India F train-other-500, avg intoantion, timbre + 741, # US Nick Marsh M train-other-500, intoantion, timbre, old man voice, unique voice +] +# chunk 5 +speaker_ids5 = [ + 922, # ricell + 1000, # artos + 1007, # Mike Conrad + 858, # Scott Merrill + 943, # Matthew C. Heckel + 984, # woggy298 + 936, # BUAES + 935, # Topaz + 977, # Logan McCamon + 946, # Cantor + 1030, # Ancient mariner + 1046, # Preston McConkie + 1022, # peac + 908, # Quentin Manuel + 924, # Andrew Coleman + 964, # Utek + 950, # davechase + 1020, # nihilist00 + 1043, # B. G. Oxford + 881, # mpetranech + 852, # Steven Proctor + 995, # Parrot + 1045, # joi + 1048, # tornadogrrrl + 900, # peaceuntoyou + 932, # Raerity + 1005, # Beatrice + 851, # Jennifer Lott + 897, # Jan Dawn Doronila + 1041, # mjbrichant + 863, # K Hindall + 937, # Sarah Gutierrez + 1049, # Diana Solomon + 1001, # TriciaG + 934, # Darla + 947, # Larissa Little + 944, # Sarafina Suransky + 870, # Barbara Bulkeley + 923, # Jane Greensmith + 1047, # Hannah Dowell + 967, # Stephanie Land + 929, # Petra + 963, # MichelleHarris + 891, # anoldfashiongirl + 890, # PopularOutcast + 992, # Fran +] +# chunk_6 +speaker_ids6 = [ + 1143, # Keith Henige M train-other-500 + 1159, # Matt Wills M train-clean-360 + 1127, # C.J. Casey M train-other-500 + 1235, # Richard Kilmer M train-other-500 + 1092, # BenW M train-other-500 + 1270, # Brendan Tannam M train-other-500 + 1214, # David Baldwin M train-clean-360 + 1255, # Daniel Paashaus M train-other-500 + 1152, # Brian Keith Barnes M train-clean-360 + 1158, # StarrDog M train-other-500 + 1256, # Graeme Dunlop M train-other-500 + 1215, # Kevin Maxson M train-clean-360 + 1274, # Jud Niven M train-clean-360 + 1168, # Epistomolus M train-clean-100 + 1089, # Bill Ruhsam M train-clean-360 + 1142, # Jonathan Burchard M train-other-500 + 1090, # Termin Dyan M train-other-500 + 1109, # Martin Geeson M train-other-500 + 1230, # Troy Bond M train-other-500 + 1150, # TexasSteve M train-other-500 + 1191, # Denise Lacey F train-other-500 + 1259, # Megan Argo F train-other-500 + 1238, # madmouth F train-other-500 + 1135, # Linda Andrus F train-clean-360 + 1247, # Sarah LuAnn F train-clean-100 + 1115, # Rachel Gatwood F train-clean-360 + 1065, # Bob Sherman M train-other-500 + 1204, # Dale A. Bade F train-clean-360 + 1174, # Frances Marcinkiewicz F train- + 1257, # Availle F train-other-500 + 1239, # Rachell Lovett F train-clean-360 + 1273, # gmiteva F train-other-500 + 1242, # Richard Ellwood M train-clean-360 + 1093, # Katie Riley F train-clean-360 + 1063, # SuD F train-other-500 + 1098, # Kerry Hiles F train-other-500 + 1254, # Rosie F train-clean-100 + 1157, # Bev J Stevens F train-clean-360 + 1184, # Joseph Couves F train-other-500 + 1253, # Caroline Shapiro F train-other-500 + 1183, # Evelyn Clarke F train-other-500 + 1082, # Symmie F train-clean-360 + 1128, # Linda Ferguson F train-other-500 + 1108, # Paul McCartan M train-other-500 + 1202, # Joy Easton F train-clean-360 + 1226, # serenitylee F train-clean-360 + 1105, # Bridget Gaige F train-clean-360 + 1229, # RoseA F train-clean-360 + 1181, # J. Rebecca Franklin F train-clean-360 + 1231, # Abigail Bartels F train-other-500 + 1182, # tabithat F train-other-500 + 1217, # JimOCR M train-other-500 + 1171, # Roberta Carlisle F train-other-500 + 1268, # A. Janelle Risa F train-clean-100 + 1243, # Rachel P. F train-clean-360 + 1071, # js392 F train-other-500 +] +# chunk_7 +speaker_ids7 = [ + 1428, # Gary Dzierlenga + 1315, # John Dennison + 1376, # mevans + 1330, # William Peck + 1400, # scrawl + 1314, # Michael Wolf + 1425, # Jonah Cummings + 1438, # Tom Barron + 1281, # garbageman99 + 1414, # Preston Scrape + 1375, # Frank Adams + 1410, # Zachary Johnson + 1365, # Eric Leach + 1302, # davidb + 1354, # Kristen Zaza + 1346, # Jeanie + 1320, # Anita Fleming + 1370, # Savanna Herrold + 1290, # Veronica Jenkins + 1437, # Charles RUHE + 1297, # May Low + 1440, # P Moscato + 1433, # browneyedgirl32382 + 1366, # cher0520 + 1285, # Ric F + 1399, # Jeanne Luft + 1402, # Angel5 + 1303, # kiwafruit + 1301, # Barbara Clements + 1453, # Anna-Lisa Ott + 1374, # pachayes + 1373, # Maria Therese +] +# chunk 8 +speaker_ids8 = [ + 1649, # Phineas Redux + 1691, # sparks0314 + 1672, # Mike Wajda + 1539, # Nathan Jordan + 1610, # jgoffena + 1512, # Matt Soar + 1526, # Mike Harris + 1647, # Patrick Reinhart + 1636, # jessecoy + 1676, # Gargoyle + 1595, # Matthew Reece + 1609, # Jacob Paul Starr + 1671, # bobbybrill + 1555, # Andrew Nelson + 1657, # alwpoe + 1592, # jerryB + 1505, # Rom Maczka + 1565, # bryan.peterson + 1644, # Christopher Maust + 1695, # Tina Nuzzi + 1702, # Sirmelja + 1697, # Lonelle Yoder + 1596, # Joyce Couch + 1660, # Jerry Romero + 1524, # Elizabeth Barr + 1643, # Linette Geisel + 1543, # Lauren McCullough + 1613, # Elsa Youngsteadt + 1662, # GabrielleC + 1587, # Claudia Wilson + 1641, # Kirsten Wever + 1614, # Jennifer Dionne + 1603, # Christine Rodriguez + 1546, # Carrie Heyes + 1579, # Linda Velwest + 1638, # Laura Victoria + 1651, # Debbie Pieterse + 1554, # Natalie Sullivan + 1656, # Sharon Omi + 1607, # Lynda Sizemore + 1670, # dmbrought + 1659, # kelleywyskiel +] +# chunk_9 +speaker_ids9 = [ + 1785, # BR TRUEBRIT M train-other-500, timbre, min intonation + 1776, # US dan_h M train-other-500, avg intoantion, timbre + 1791, # US David Cummings M train-clean-360, avg intoantion, timbre + 1707, # US David Olson M train-other-500, unique voice, intoantion, timbre + 1801, # US John O M train-other-500, unique voice, min intoantion, timbre + 1903, # US stephenreader M train-other-500, velvet unique voice + 1856, # BR Craig Gulliver M train-other-500, avg intoantion, timbre + 1906, # US Tony Russell M train-clean-360, dictor voice, avg intoantion, timbre + 1806, # US Marc Pizzuti M train-other-500, avg intoantion + 1828, # US David Wales M train-clean-100, avg intoantion, timbre + 1849, # US Fred DeBerardinis M train-clean-100, avg intoantion, timbre, unique voice + 1712, # BR Steve Belleguelle M train-other-500, avg intoantion, timbre + 1853, # US Ron Altman M train-other-500, intoantion, unique voice, timbre + 1805, # US Vince Dee M train-clean-100, unique voice, timbre + 1838, # US Morey Kunin M train-clean-360, young voice, intoantion, timbre + 1862, # US Alexandre Laplante M train-clean-360, young voice, intoantion, timbre + 1915, # US T.E. McHenry M train-other-500, unique voice, dictor voice, intoantion, timbre + 1772, # US haggisreflux M train-clean-360, intoantion, timbre + 1831, # BR Nigel Boydell M train-other-500, unique voice, intoantion, timbre + 1706, # BR Deborah Knight F train-clean-100, unique voice, intoantion, timbre + 1908, # US fshort F train-other-500 + 1783, # US Sarah Crampton F train-clean-360 + 1781, # US humanode M train-clean-360, unique voice + 1779, # US Wendy Almeida F train-clean-360 + 1839, # US James E. Carson M train-clean-360 + 1724, # US Juliana M. F train-clean-360 + 1728, # US Vinnie Tesla M train-clean-360 + 1907, # US Snapdragon F train-other-500 + 1881, # US Julienne F train-other-500 + 1802, # US selway F train-other-500 + 1826, # US John Hoerr M train-clean-100 + 1725, # BR Ruth Kidson F train-other-500 + 1764, # US ReadWriteLib F train-clean-360 + 1794, # US Michelle Remington F train-clean-360 + 1880, # US Christine Nendza F train-clean-360 + 1848, # US Monica Knuppe F train-clean-360 + 1736, # US Spike Holcomb F train-other-500 + 1841, # US Elena F train-clean-360 + 1836, # US Kendall Ashyby F train-other-500 + 1741, # US anjieliu F train-other-500 + 1803, # US Susan Hanfield F train-clean-360 + 1761, # US EliMarieHK F train-other-500 + 1745, # US Janet F train-clean-360 + 1713, # US dobsonfly F train-clean-100 + 1716, # US EyeBones F train-clean-360 + 1814, # US polkadotish F train-other-500 + 1709, # US CrowGirl F train-other-500 + 1763, # US Gen Jones F train-clean-360 + 1808, # US Rebecca King F train-clean-360 + 1811, # US Michelle Day F train-clean-360 + 1857, # US Amanda Friday F train-clean-360 + 1893, # US KirksVoice M train-other-500 + 1820, # US Feyaza F train-other-500 + 1771, # US Chelsea S. F train-other-500 + 1718, # US Caroline Driggs F train-other-500 + 1752, # US Shana Cohen F train-clean-360 + 1869, # US NastassiaS F train-other-500 + 1863, # US Tika Sabu F train-other-500 + 1723, # US Rachel Bossier F train-other-500 + 1798, # US C. L. W. Rollins F train-other-500 + 1755, # US Yvonne Smith F train-clean-360 + 1738, # US Lois C. Johnson F train-clean-360 + 1887, # US Jenna Lanman F train-clean-360 +] +# chunk 10/11 +speaker_ids10_11 = [ + 1956, # Thomas Meaney + 2049, # AdrianBisson + 1978, # John Trevithick + 2001, # Wesseling + 2114, # Larry Beasley + 2032, # doonaboon + 2087, # James Bendall + 2011, # pekein + 2056, # acloward + 2007, # Art Leung + 2084, # Eberle Thomas + 2115, # Pete Milan + 1987, # Andrew White + 1959, # DVoice + 1954, # Szindbad + 2036, # T.K. Kirven + 1947, # Barbara Edelman + 2045, # Linda Ciano + 1979, # roeg11 + 2075, # Joy S Grape + 2091, # Caroline Hemmerly Kunkle + 2023, # Vickie Ranz + 2014, # Eden Rea-Hedrick + 1965, # redhed3095 + 1989, # Joannemmp + 2040, # MJ Franck + 1996, # Mary in Arkansas + 1957, # Sarika Pawar + 2100, # Katherine + 2069, # Asta1234 + 2096, # Tara Dow + 2095, # Diana Dolan + 1995, # Jill Janovetz + 2017, # CaprishaPage + 2010, # Peggy + 1998, # voicebynatalie + 1952, # Katalina Watt + 2094, # Meg Cowan + 2065, # Muriel + 2312, # Jon Kerfoot + 2217, # Jesse Crisp-Sears + 2197, # Mike Nelson + 2282, # Robert Snoza + 2192, # Sammy Bean + 2268, # Greg Giordano + 2278, # Jake Woldstad + 2241, # Steven Reynolds + 2239, # amaskill + 2225, # nomorejeffs + 2283, # Tim Cote + 2230, # Sam Naishtat + 2151, # MaxSitting + 2141, # KateC + 2314, # Cheri Jordan + 2127, # Ron Lockhart + 2147, # Shawn Bayern + 2251, # Wiley Combs + 2195, # Lynne Thompson + 2272, # JamesMcAndrew + 2156, # C F de Rosset + 2292, # Arnold + 2143, # Suebee + 2333, # Anita Slusser + 2233, # Alexis Castro + 2305, # Brooke Cunningham + 2247, # Lois Browne + 2171, # Carolyne + 2172, # Demosthenes + 2291, # lewildesen + 2194, # Iridescat + 2331, # Madam Fickle + 2317, # helengraves + 2234, # Coreena + 2209, # Samantha J Gubitz + 2152, # Kristel Tretter + 2267, # Frances Brown + 2275, # NatalieOram + 2298, # Sheila Wood + 2138, # Jeannie Tirado + 2220, # Loveday +] +# chunk_12 +speaker_ids12 = [ + 2403, # Ian Quinlan M train-clean-360 + 2436, # IND josembi M train-other-500 + 2387, # Brett G. Hirsch M train-other-500 + 2444, # dsilber01 M train-clean-360 + 2419, # Gary Dana M train-clean-100 + 2453, # Krzysztof Rowinski M train-clean-360 + 2451, # DeanOBuchanan M train-clean-100 + 2473, # Eric Metzler M train-clean-360 + 2415, # Patrick Eaton M train-other-500 + 2379, # pjhoury M train-other-500 + 2377, # Jon Kissack M train-clean-100 + 2355, # yeknod M train-other-500 + 2452, # Walt Allan M train-other-500 + 2401, # Matt Parker M train-clean-360 + 2359, # Doug Reed M train-other-500 + 2425, # noblesavage M train-clean-100 + 2390, # sdaeley17 M train-clean-360 + 2461, # ScottReyonoldsVoice M train-clean-360 + 2371, # Alexander Hatton M train-clean-360 + 2479, # Daisy Flaim F train-clean-100 + 2483, # Tammy Porter F train-clean-360 + 2372, # Lynne Ray F train-clean-360 + 2422, # Jude Somers F train-clean-360 + 2357, # William Gavula M train-other-500 + 2439, # KHand F train-clean-360 + 2441, # Alison Stewart F train-clean-360 + 2413, # Joanne Rochon F train-clean-360 + 2383, # Emma Joyce F train-other-500 + 2378, # Jackie Drown F train-clean-360 + 2352, # Jaimie Noy F train-clean-100 + 2397, # Rebecca Braunert-Plunkett F train-other-500 + 2394, # TinaNygard2 F train-clean-100 + 2447, # Deena Rhoads F train-clean-360 + 2358, # Betty Perry F train-clean-360 + 2471, # MariaS F train-other-500 + 2468, # Erin Schellhase F train-clean-360 + 2370, # gloriousjob M train-clean-360 + 2341, # Haili F train-other-500 + 2469, # Kevin Owens M train-clean-100 + 2448, # Emily Maynard F train-clean-360 + 2351, # Nick Bulka M train-other-500 +] + +speaker_ids = speaker_ids1 + speaker_ids2 + speaker_ids3 + speaker_ids4 + speaker_ids5 + speaker_ids6 + speaker_ids7 + speaker_ids8 + speaker_ids9 + speaker_ids10_11 + speaker_ids12 + +# Selected for the fine-tuning +selected_speakers = [ + 574, # Daniel Shorten M train-clean-100 + 242, # J. Hall M train-other-500 + 536, # Robert Flach M train-other-500 + 82, # Andy Minter M train-other-500 + 672, # Stuart Bell M train-other-500 + 315, # Jean Crevier M train-other-500 + 628, # Bryan Ness M train-clean-100 + 61, # John Greenman M train-other-500 + 649, # Scarlett! F train-clean-360 + 105, # Marian Brown F train-clean-360 + 399, # entada F train-clean-360 + 89, # Paula Berinstein F train-clean-360 + 502, # Lee Elliott F train-other-500 + 102, # Maureen S. O'Brien F train-clean-100 + 544, # Miranda Stinson F train-clean-360 + 653, # cucciasv F train-other-500 + 465, # Leonie Rose F train-clean-100 + 96, # Kymm Zuckert F train-other-500 + 447, # Lee Ann Howlett F train-clean-360 + 165, # Elisabeth Shields F train-clean-100 + 430, # Millbeach F train-other-500 + 214, # Scott Splavec M train-clean-100 + 666, # Kelly Dougherty M train-clean-360 + 481, # Scott Sherris M train-clean-360 + 463, # Chris Hughes M train-other-500 + 273, # Andrew Lebrun M train-other-500 + 172, # Harvey Chinn M train-other-500 + 83, # Graham Williams M train-other-500 + 523, # Michael Loftus M train-clean-360 + 38, # Kurt Copeland M train-clean-360 + 248, # fieldsofgold M train-other-500 + 234, # Menno M train-other-500 + 145, # Mr. Baby Man M train-clean-360 + 250, # Quentin M train-clean-360 + 498, # Chris Gladis M train-clean-100 + 123, # Sean McGaughey M train-clean-360 + 171, # Paul Harvey M train-clean-360 + 49, # Kristen McQuillin F train-clean-100 + 588, # Kalynda F train-clean-360 + 117, # Caitlin Kelly F train-clean-360 + 657, # Shannon F train-other-500 + 275, # Zale Schafer (Rose May Chamberlin Memorial Foundat F train-clean-360 + 604, # Anne-Marie F train-other-500 + 64, # Christiane Levesque F train-clean-360 + 685, # Nikki Sullivan F train-clean-100 + 355, # Lana Taylor F train-clean-100 + 185, # Kim Braun F train-clean-360 + 52, # Cori Samuel F train-other-500 + 218, # Joy Chan F train-other-500 + 549, # AmyAG F train-other-500 + 617, # PJ F train-other-500 + 414, # Christabel F train-clean-100 + 382, # Kelli Robinson F train-clean-360 + 76, # ML Cohen M train-other-500 + 176, # Micah Sheppard M train-clean-360 + 233, # mikenkat M train-clean-360 + 390, # JimmyLogan M train-clean-360 + 393, # Tim Lundeen M train-clean-360 + 425, # RedToby M train-clean-360 + 398, # Sam Fold M train-other-500 + 372, # Jim Mullins M train-clean-360 + 99, # Stewart Wills M train-clean-100 + 340, # Nick Gallant M train-clean-100 + 40, # JemmaBlythe F train-other-500 + 118, # Brenda Dayne F train-clean-360 + 640, # David A. Stokely M train-other-500 + 50, # Dan Threetrees M train-clean-360 + 373, # Brooks Seveer M train-clean-360 + 124, # Steve Karafit M train-clean-100 + 314, # Carl Vonnoh, III M train-clean-360 + 531, # Fr. Richard Zeile of Detroit M train-other-500 + 383, # Mike Roop M train-other-500 + 710, # Sheila Morton F train-clean-100 + 450, # Heather Duncan F train-clean-360 + 645, # Micah M train-other-500 + 517, # Madame Tusk F train-other-500 + 479, # Wina Hathaway F train-other-500 + 30, # Ophelia Darcy F train-other-500 + 220, # Tina Tilney F train-clean-360 + 63, # Linda Wilcox F train-other-500 + 283, # Bethany Simpson F train-clean-360 + 644, # Cynthia Zocca F train-clean-360 + 677, # Allyson Hester F train-other-500 + 21, # Kelly Bescherer F train-other-500 + 552, # Mim Ritty F train-clean-100 + 80, # Fox in the Stars F train-clean-100 + 394, # swroot F train-clean-360 + 426, # Megan Stemm-Wade F train-clean-100 + 91, # Chris Goringe M train-other-500 + 108, # Kevin McAsh M train-clean-360 + 130, # Peter of Buckinghamshire England M train-other-500 + 661, # James Gladwin M train-other-500 + 216, # Dave Ranson M train-clean-100 + 164, # Ed Good M train-other-500 + 308, # Eric Connover M train-other-500 + 569, # Arouet M train-clean-360 + 313, # Tim Bulkeley M train-other-500 + 212, # Glen Hallstrom M train-other-500 + 15, # Chip M train-other-500 + 469, # Christian Pecaut M train-clean-360 + 294, # Diana Kiesners F train-clean-360 + 192, # Nocturna F train-clean-100 + 73, # Claire Goget F train-clean-100 + 417, # Kiki Baessell F train-clean-360 + 636, # Matthew Howell F train-other-500 + 36, # chriss the girl F train-other-500 + 668, # Jan Baxter F train-clean-360 + 403, # Igor Teaforay F train-clean-360 + 618, # Linnea F train-other-500 + 596, # Jo F train-other-500 + 499, # Tammy Sanders F train-clean-100 + 207, # Sage Tyrtle F train-other-500 + 1346, # Jeanie F train-other-500 + 1109, # Martin Geeson M train-other-500 + 770, # Pete Williams, Pittsburgh, PA M train-clean-360 + 1247, # Sarah LuAnn F train-clean-100 + 1526, # Mike Harris M train-other-500 + 908, # Quentin Manuel M train-clean-360 + 1183, # Evelyn Clarke F train-other-500 + 1438, # Tom Barron M train-other-500 + 1022, # peac M train-clean-100 + 1603, # Christine Rodriguez F train-clean-360 + 1425, # Jonah Cummings M train-clean-360 + 731, # Priya, India F train-other-500 + 782, # Alec Daitsman M train-clean-360 + 1090, # Termin Dyan M train-other-500 + 995, # Parrot M train-other-500 + 923, # Jane Greensmith F train-clean-360 + 766, # Clive Catterall M train-other-500 + 822, # kristiface F train-clean-360 + 897, # Jan Dawn Doronila F train-clean-360 + 1579, # Linda Velwest F train-clean-360 + 964, # Utek M train-clean-360 + 1414, # Preston Scrape M train-other-500 + 834, # Serin F train-other-500 + 1302, # davidb M train-clean-360 + 1135, # Linda Andrus F train-clean-360 + 1440, # P Moscato F train-clean-360 + 870, # Barbara Bulkeley F train-clean-360 + 1256, # Graeme Dunlop M train-other-500 + 1255, # Daniel Paashaus M train-other-500 + 1157, # Bev J Stevens F train-clean-360 + 934, # Darla F train-other-500 + 1281, # garbageman99 M train-clean-360 + 819, # n8evv M train-clean-360 + 1041, # mjbrichant F train-other-500 + 863, # K Hindall F train-clean-360 + 1303, # kiwafruit F train-clean-100 + 1115, # Rachel Gatwood F train-clean-360 + 1539, # Nathan Jordan M train-other-500 + 1428, # Gary Dzierlenga M train-other-500 + 1049, # Diana Solomon F train-other-500 + 1546, # Carrie Heyes F train-other-500 + 1089, # Bill Ruhsam M train-clean-360 + 1142, # Jonathan Burchard M train-other-500 + 1375, # Frank Adams M train-clean-360 + 881, # mpetranech M train-other-500 + 798, # Wyatt M train-other-500 + 1647, # Patrick Reinhart M train-clean-360 + 1587, # Claudia Wilson F train-clean-360 + 830, # musici123 F train-other-500 + 1592, # jerryB M train-other-500 + 839, # Ben Dutton M train-other-500 + 835, # Rachel Lintern F train-other-500 + 1273, # gmiteva F train-other-500 + 932, # Raerity F train-other-500 + 1108, # Paul McCartan M train-other-500 + 732, # Tysto M train-clean-360 + 781, # Megan Kunkel F train-other-500 + 1555, # Andrew Nelson M train-clean-360 + 1437, # Charles RUHE M train-clean-360 + 1402, # Angel5 F train-other-500 + 963, # MichelleHarris F train-clean-360 + 1181, # J. Rebecca Franklin F train-clean-360 + 818, # Matt Warzel F train-clean-360 + 1285, # Ric F M train-clean-100 + 797, # Chris Jones F train-other-500 + 1505, # Rom Maczka M train-clean-360 + 1214, # David Baldwin M train-clean-360 + 1636, # jessecoy M train-other-500 + 929, # Petra F train-other-500 + 1171, # Roberta Carlisle F train-other-500 + 817, # texttalker M train-clean-360 + 1433, # browneyedgirl32382 F train-clean-360 + 1158, # StarrDog M train-other-500 + 1000, # artos M train-other-500 + 848, # senshisteph F train-other-500 + 1596, # Joyce Couch F train-other-500 + 757, # Roger Melin M train-clean-360 + 1168, # Epistomolus M train-clean-100 + 741, # Nick Marsh M train-other-500 + 1649, # Phineas Redux M train-other-500 + 851, # Jennifer Lott F train-clean-360 + 808, # M. J. Boyle F train-other-500 + 1595, # Matthew Reece M train-clean-360 + 1370, # Savanna Herrold F train-other-500 + 1565, # bryan.peterson M train-other-500 + 944, # Sarafina Suransky F train-other-500 + 1268, # A. Janelle Risa F train-clean-100 + 771, # Isosceles F train-clean-360 + 752, # Cat Schirf F train-other-500 + 800, # Jack Farrell M train-clean-360 + 1005, # Beatrice F train-other-500 + 1229, # RoseA F train-clean-360 + 943, # Matthew C. Heckel M train-clean-360 + 891, # anoldfashiongirl F train-other-500 + 1226, # serenitylee F train-clean-360 + 1253, # Caroline Shapiro F train-other-500 + 1204, # Dale A. Bade F train-clean-360 + 1230, # Troy Bond M train-other-500 + 791, # David Kleparek M train-clean-100 + 1184, # Joseph Couves F train-other-500 + 1001, # TriciaG F train-clean-360 + 804, # FirstKnight F train-other-500 + 1641, # Kirsten Wever F train-clean-100 + 1259, # Megan Argo F train-other-500 + 1231, # Abigail Bartels F train-other-500 + 1410, # Zachary Johnson M train-other-500 + 1030, # Ancient mariner M train-other-500 + 1093, # Katie Riley F train-clean-360 + 1254, # Rosie F train-clean-100 + 1365, # Eric Leach M train-clean-360 + 831, # David Federman M train-other-500 + 1989, # Joannemmp F train-clean-100 + 1707, # David Olson M train-other-500 + 1849, # Fred DeBerardinis M train-clean-100 + 1808, # Rebecca King F train-clean-360 + 2292, # Arnold M train-clean-100 + 2415, # Patrick Eaton M train-other-500 + 1656, # Sharon Omi F train-clean-100 + 1676, # Gargoyle M train-clean-360 + 1881, # Julienne F train-other-500 + 2036, # T.K. Kirven F train-other-500 + 1761, # EliMarieHK F train-other-500 + 2115, # Pete Milan M train-other-500 + 1803, # Susan Hanfield F train-clean-360 + 1798, # C. L. W. Rollins F train-other-500 + 1723, # Rachel Bossier F train-other-500 + 2341, # Haili F train-other-500 + 2468, # Erin Schellhase F train-clean-360 + 1725, # Ruth Kidson F train-other-500 + 2010, # Peggy F train-other-500 + 1853, # Ron Altman M train-other-500 + 2359, # Doug Reed M train-other-500 + 2422, # Jude Somers F train-clean-360 + 2234, # Coreena F train-other-500 + 2156, # C F de Rosset F train-other-500 + 2483, # Tammy Porter F train-clean-360 + 1781, # humanode M train-clean-360 + 2275, # NatalieOram F train-other-500 + 2390, # sdaeley17 M train-clean-360 + 2314, # Cheri Jordan F train-clean-360 + 2413, # Joanne Rochon F train-clean-360 + 1697, # Lonelle Yoder F train-other-500 + 1718, # Caroline Driggs F train-other-500 + 2387, # Brett G. Hirsch M train-other-500 + 2331, # Madam Fickle F train-clean-100 + 1783, # Sarah Crampton F train-clean-360 + 2397, # Rebecca Braunert-Plunkett F train-other-500 + 2357, # William Gavula M train-other-500 + 1670, # dmbrought M train-other-500 + 1987, # Andrew White M train-clean-360 + 1755, # Yvonne Smith F train-clean-360 + 2192, # Sammy Bean M train-other-500 + 1716, # EyeBones F train-clean-360 + 1828, # David Wales M train-clean-100 + 2251, # Wiley Combs M train-clean-360 + 2065, # Muriel F train-clean-360 + 2017, # CaprishaPage F train-other-500 + 1947, # Barbara Edelman F train-other-500 + 1738, # Lois C. Johnson F train-clean-360 + 1791, # David Cummings M train-clean-360 + 2045, # Linda Ciano F train-clean-360 + 2452, # Walt Allan M train-other-500 + 2040, # MJ Franck F train-other-500 + 1831, # Nigel Boydell M train-other-500 + 2371, # Alexander Hatton M train-clean-360 + 1954, # Szindbad M train-other-500 + 1836, # Kendall Ashyby F train-other-500 + 2436, # josembi M train-other-500 + 2383, # Emma Joyce F train-other-500 + 2278, # Jake Woldstad M train-clean-360 + 1741, # anjieliu F train-other-500 + 1857, # Amanda Friday F train-clean-360 + 2370, # gloriousjob M train-clean-360 + 1907, # Snapdragon F train-other-500 + 2225, # nomorejeffs M train-clean-360 + 2439, # KHand F train-clean-360 + 2239, # amaskill M train-other-500 + 2007, # Art Leung F train-clean-360 + 2283, # Tim Cote M train-clean-360 + 1712, # Steve Belleguelle M train-other-500 + 2094, # Meg Cowan F train-clean-360 + 1772, # haggisreflux M train-clean-360 + 2317, # helengraves F train-clean-360 + 2241, # Steven Reynolds M train-clean-360 + 2011, # pekein M train-clean-360 + 1826, # John Hoerr M train-clean-100 + 1695, # Tina Nuzzi F train-clean-360 + 2451, # DeanOBuchanan M train-clean-100 + 1771, # Chelsea S. F train-other-500 + 2441, # Alison Stewart F train-clean-360 + 1745, # Janet F train-clean-360 + 2358, # Betty Perry F train-clean-360 + 2197, # Mike Nelson M train-other-500 + 2014, # Eden Rea-Hedrick F train-other-500 + 1672, # Mike Wajda M train-clean-360 + 2394, # TinaNygard2 F train-clean-100 + 1657, # alwpoe M train-clean-360 + 1728, # Vinnie Tesla M train-clean-360 + 1805, # Vince Dee M train-clean-100 + 2143, # Suebee F train-clean-360 + 2084, # Eberle Thomas M train-other-500 + 2479, # Daisy Flaim F train-clean-100 + 2152, # Kristel Tretter F train-clean-360 + 2268, # Greg Giordano M train-clean-360 + 1839, # James E. Carson M train-clean-360 + 2056, # acloward M train-clean-360 + 1814, # polkadotish F train-other-500 + 2127, # Ron Lockhart M train-clean-100 + 2114, # Larry Beasley M train-clean-360 + 2469, # Kevin Owens M train-clean-100 + 2447, # Deena Rhoads F train-clean-360 + 1724, # Juliana M. F train-clean-360 + 1869, # NastassiaS F train-other-500 + 2209, # Samantha J Gubitz F train-clean-360 + 2171, # Carolyne F train-other-500 + 2403, # Ian Quinlan M train-clean-360 + 2032, # doonaboon M train-other-500 + 2075, # Joy S Grape F train-clean-360 +] diff --git a/config/latest_selection.txt b/config/latest_selection.txt new file mode 100644 index 0000000000000000000000000000000000000000..52aedae30fa470053fae63bac7a6c22305642a15 --- /dev/null +++ b/config/latest_selection.txt @@ -0,0 +1,331 @@ +id name Nikita Andrey Alexey Result OR Result AND Count Yes > 1 +574 Daniel Shorten Yes No Yes TRUE FALSE TRUE +242 J. Hall No No No FALSE FALSE FALSE +536 Robert Flach Yes No No TRUE FALSE FALSE +82 Andy Minter No No No FALSE FALSE FALSE +672 Stuart Bell Yes No No TRUE FALSE FALSE +315 Jean Crevier Yes No No TRUE FALSE FALSE +628 Bryan Ness Yes No No TRUE FALSE FALSE +61 John Greenman No No No FALSE FALSE FALSE +649 Scarlett! Yes Yes Yes TRUE TRUE TRUE +105 Marian Brown No No No FALSE FALSE FALSE +399 entada No No No FALSE FALSE FALSE +89 Paula Berinstein No No Yes TRUE FALSE FALSE +502 Lee Elliott No No No FALSE FALSE FALSE +102 Maureen S. O'Brien Yes Yes No TRUE FALSE TRUE +544 Miranda Stinson Yes Yes Yes TRUE TRUE TRUE +653 cucciasv Yes Yes Yes TRUE TRUE TRUE +465 Leonie Rose No No No FALSE FALSE FALSE +96 Kymm Zuckert Yes No No TRUE FALSE FALSE +447 Lee Ann Howlett No No No FALSE FALSE FALSE +165 Elisabeth Shields Yes No No TRUE FALSE FALSE +430 Millbeach No No No FALSE FALSE FALSE +214 Scott Splavec No Yes No TRUE FALSE FALSE +666 Kelly Dougherty Yes Yes No TRUE FALSE TRUE +481 Scott Sherris Yes No Yes TRUE FALSE TRUE +463 Chris Hughes No No No FALSE FALSE FALSE +273 Andrew Lebrun Yes No No TRUE FALSE FALSE +172 Harvey Chinn Yes No No TRUE FALSE FALSE +83 Graham Williams No No No FALSE FALSE FALSE +523 Michael Loftus No No No FALSE FALSE FALSE +38 Kurt Copeland No No No FALSE FALSE FALSE +248 fieldsofgold Yes Yes No TRUE FALSE TRUE +234 Menno No No No FALSE FALSE FALSE +145 Mr. Baby Man Yes No No TRUE FALSE FALSE +250 Quentin No No No FALSE FALSE FALSE +498 Chris Gladis No No No FALSE FALSE FALSE +123 Sean McGaughey Yes Yes No TRUE FALSE TRUE +171 Paul Harvey Yes Yes No TRUE FALSE TRUE +49 Kristen McQuillin No No No FALSE FALSE FALSE +588 Kalynda No No No FALSE FALSE FALSE +117 Caitlin Kelly Yes No No TRUE FALSE FALSE +657 Shannon No No No FALSE FALSE FALSE +275 Zale Schafer (Rose May Chamberlin Memorial Foundat No No No FALSE FALSE FALSE +604 Anne-Marie Yes Yes Yes TRUE TRUE TRUE +64 Christiane Levesque Yes Yes No TRUE FALSE TRUE +685 Nikki Sullivan Yes No Yes TRUE FALSE TRUE +355 Lana Taylor No No No FALSE FALSE FALSE +185 Kim Braun No No No FALSE FALSE FALSE +52 Cori Samuel Yes No Yes TRUE FALSE TRUE +218 Joy Chan Yes Yes Yes TRUE TRUE TRUE +549 AmyAG No No No FALSE FALSE FALSE +617 PJ Yes Yes Yes TRUE TRUE TRUE +414 Christabel Yes No Yes TRUE FALSE TRUE +382 Kelli Robinson No No No FALSE FALSE FALSE +76 ML Cohen No No Yes TRUE FALSE FALSE +176 Micah Sheppard Yes No No TRUE FALSE FALSE +233 mikenkat No Yes No TRUE FALSE FALSE +390 JimmyLogan Yes No No TRUE FALSE FALSE +393 Tim Lundeen No No No FALSE FALSE FALSE +425 RedToby Yes No Yes TRUE FALSE TRUE +398 Sam Fold No No No FALSE FALSE FALSE +372 Jim Mullins Yes No No TRUE FALSE FALSE +99 Stewart Wills No No No FALSE FALSE FALSE +340 Nick Gallant No No No FALSE FALSE FALSE +40 JemmaBlythe No No No FALSE FALSE FALSE +118 Brenda Dayne Yes Yes Yes TRUE TRUE TRUE +640 David A. Stokely Yes No No TRUE FALSE FALSE +50 Dan Threetrees Yes No Yes TRUE FALSE TRUE +373 Brooks Seveer Yes Yes No TRUE FALSE TRUE +124 Steve Karafit No No No FALSE FALSE FALSE +314 Carl Vonnoh, III Yes Yes Yes TRUE TRUE TRUE +531 Fr. Richard Zeile of Detroit Yes No No TRUE FALSE FALSE +383 Mike Roop Yes No No TRUE FALSE FALSE +710 Sheila Morton Yes No Yes TRUE FALSE TRUE +450 Heather Duncan Yes No Yes TRUE FALSE TRUE +645 Micah Yes Yes No TRUE FALSE TRUE +517 Madame Tusk Yes Yes No TRUE FALSE TRUE +479 Wina Hathaway Yes No No TRUE FALSE FALSE +30 Ophelia Darcy No No No FALSE FALSE FALSE +220 Tina Tilney No No No FALSE FALSE FALSE +63 Linda Wilcox Yes Yes No TRUE FALSE TRUE +283 Bethany Simpson Yes No No TRUE FALSE FALSE +644 Cynthia Zocca Yes Yes No TRUE FALSE TRUE +677 Allyson Hester No No No FALSE FALSE FALSE +21 Kelly Bescherer Yes No No TRUE FALSE FALSE +552 Mim Ritty No No No FALSE FALSE FALSE +80 Fox in the Stars Yes No Yes TRUE FALSE TRUE +394 swroot Yes Yes No TRUE FALSE TRUE +426 Megan Stemm-Wade Yes No No TRUE FALSE FALSE +91 Chris Goringe Yes No Yes TRUE FALSE TRUE +108 Kevin McAsh Yes No Yes TRUE FALSE TRUE +130 Peter of Buckinghamshire England Yes No No TRUE FALSE FALSE +661 James Gladwin Yes Yes Yes TRUE TRUE TRUE +216 Dave Ranson No No No FALSE FALSE FALSE +164 Ed Good Yes Yes Yes TRUE TRUE TRUE +308 Eric Connover Yes Yes Yes TRUE TRUE TRUE +569 Arouet Yes No No TRUE FALSE FALSE +313 Tim Bulkeley No No No FALSE FALSE FALSE +212 Glen Hallstrom No No No FALSE FALSE FALSE +15 Chip Yes No No TRUE FALSE FALSE +469 Christian Pecaut Yes Yes No TRUE FALSE TRUE +294 Diana Kiesners No No No FALSE FALSE FALSE +192 Nocturna Yes Yes No TRUE FALSE TRUE +73 Claire Goget Yes No No TRUE FALSE FALSE +417 Kiki Baessell Yes No Yes TRUE FALSE TRUE +636 Matthew Howell Yes No No TRUE FALSE FALSE +36 chriss the girl Yes No No TRUE FALSE FALSE +668 Jan Baxter Yes Yes Yes TRUE TRUE TRUE +403 Igor Teaforay No No No FALSE FALSE FALSE +618 Linnea No No No FALSE FALSE FALSE +596 Jo Yes No Yes TRUE FALSE TRUE +499 Tammy Sanders No No No FALSE FALSE FALSE +207 Sage Tyrtle No No No FALSE FALSE FALSE +1346 Jeanie No No No FALSE FALSE FALSE +1109 Martin Geeson Yes No Yes TRUE FALSE TRUE +770 Pete Williams, Pittsburgh, PA Yes No Yes TRUE FALSE TRUE +1247 Sarah LuAnn Yes Yes No TRUE FALSE TRUE +1526 Mike Harris No No No FALSE FALSE FALSE +908 Quentin Manuel Yes No Yes TRUE FALSE TRUE +1183 Evelyn Clarke Yes No No TRUE FALSE FALSE +1438 Tom Barron Yes No No TRUE FALSE FALSE +1022 peac No No No FALSE FALSE FALSE +1603 Christine Rodriguez No No No FALSE FALSE FALSE +1425 Jonah Cummings No No No FALSE FALSE FALSE +731 Priya, India Yes No No TRUE FALSE FALSE +782 Alec Daitsman Yes Yes No TRUE FALSE TRUE +1090 Termin Dyan Yes No No TRUE FALSE FALSE +995 Parrot Yes Yes No TRUE FALSE TRUE +923 Jane Greensmith Yes No Yes TRUE FALSE TRUE +766 Clive Catterall No No No FALSE FALSE FALSE +822 kristiface Yes Yes Yes TRUE TRUE TRUE +897 Jan Dawn Doronila Yes No No TRUE FALSE FALSE +1579 Linda Velwest No No No FALSE FALSE FALSE +964 Utek No No No FALSE FALSE FALSE +1414 Preston Scrape Yes No Yes TRUE FALSE TRUE +834 Serin No No No FALSE FALSE FALSE +1302 davidb Yes No Yes TRUE FALSE TRUE +1135 Linda Andrus Yes No Yes TRUE FALSE TRUE +1440 P Moscato Yes Yes No TRUE FALSE TRUE +870 Barbara Bulkeley No Yes No TRUE FALSE FALSE +1256 Graeme Dunlop Yes No No TRUE FALSE FALSE +1255 Daniel Paashaus No No No FALSE FALSE FALSE +1157 Bev J Stevens No No No FALSE FALSE FALSE +934 Darla No No Yes TRUE FALSE FALSE +1281 garbageman99 Yes No Yes TRUE FALSE TRUE +819 n8evv Yes No No TRUE FALSE FALSE +1041 mjbrichant Yes No Yes TRUE FALSE TRUE +863 K Hindall No No No FALSE FALSE FALSE +1303 kiwafruit No No No FALSE FALSE FALSE +1115 Rachel Gatwood No No No FALSE FALSE FALSE +1539 Nathan Jordan No No No FALSE FALSE FALSE +1428 Gary Dzierlenga No No No FALSE FALSE FALSE +1049 Diana Solomon No No No FALSE FALSE FALSE +1546 Carrie Heyes No No No FALSE FALSE FALSE +1089 Bill Ruhsam No No No FALSE FALSE FALSE +1142 Jonathan Burchard Yes No Yes TRUE FALSE TRUE +1375 Frank Adams Yes No No TRUE FALSE FALSE +881 mpetranech Yes No Yes TRUE FALSE TRUE +798 Wyatt No No No FALSE FALSE FALSE +1647 Patrick Reinhart No No No FALSE FALSE FALSE +1587 Claudia Wilson Yes No No TRUE FALSE FALSE +830 musici123 Yes No No TRUE FALSE FALSE +1592 jerryB No No No FALSE FALSE FALSE +839 Ben Dutton No No No FALSE FALSE FALSE +835 Rachel Lintern Yes No Yes TRUE FALSE TRUE +1273 gmiteva Yes No No TRUE FALSE FALSE +932 Raerity Yes Yes No TRUE FALSE TRUE +1108 Paul McCartan No No No FALSE FALSE FALSE +732 Tysto Yes Yes No TRUE FALSE TRUE +781 Megan Kunkel No No No FALSE FALSE FALSE +1555 Andrew Nelson No No No FALSE FALSE FALSE +1437 Charles RUHE No No No FALSE FALSE FALSE +1402 Angel5 Yes No Yes TRUE FALSE TRUE +963 MichelleHarris No No No FALSE FALSE FALSE +1181 J. Rebecca Franklin No No No FALSE FALSE FALSE +818 Matt Warzel No No No FALSE FALSE FALSE +1285 Ric F Yes No No TRUE FALSE FALSE +797 Chris Jones Yes No No TRUE FALSE FALSE +1505 Rom Maczka Yes No No TRUE FALSE FALSE +1214 David Baldwin No No No FALSE FALSE FALSE +1636 jessecoy No No No FALSE FALSE FALSE +929 Petra Yes Yes No TRUE FALSE TRUE +1171 Roberta Carlisle No No No FALSE FALSE FALSE +817 texttalker Yes Yes No TRUE FALSE TRUE +1433 browneyedgirl32382 Yes Yes Yes TRUE TRUE TRUE +1158 StarrDog Yes No No TRUE FALSE FALSE +1000 artos No No No FALSE FALSE FALSE +848 senshisteph Yes No No TRUE FALSE FALSE +1596 Joyce Couch Yes Yes No TRUE FALSE TRUE +757 Roger Melin Yes No No TRUE FALSE FALSE +1168 Epistomolus Yes No No TRUE FALSE FALSE +741 Nick Marsh Yes No No TRUE FALSE FALSE +1649 Phineas Redux No No No FALSE FALSE FALSE +851 Jennifer Lott Yes Yes Yes TRUE TRUE TRUE +808 M. J. Boyle No No No FALSE FALSE FALSE +1595 Matthew Reece No No No FALSE FALSE FALSE +1370 Savanna Herrold Yes Yes Yes TRUE TRUE TRUE +1565 bryan.peterson No Yes No TRUE FALSE FALSE +944 Sarafina Suransky Yes No No TRUE FALSE FALSE +1268 A. Janelle Risa Yes No No TRUE FALSE FALSE +771 Isosceles No No No FALSE FALSE FALSE +752 Cat Schirf No No No FALSE FALSE FALSE +800 Jack Farrell No No No FALSE FALSE FALSE +1005 Beatrice Yes No No TRUE FALSE FALSE +1229 RoseA No No No FALSE FALSE FALSE +943 Matthew C. Heckel No No No FALSE FALSE FALSE +891 anoldfashiongirl Yes No No TRUE FALSE FALSE +1226 serenitylee No No No FALSE FALSE FALSE +1253 Caroline Shapiro Yes No No TRUE FALSE FALSE +1204 Dale A. Bade Yes No Yes TRUE FALSE TRUE +1230 Troy Bond Yes Yes No TRUE FALSE TRUE +791 David Kleparek Yes Yes No TRUE FALSE TRUE +1184 Joseph Couves No No No FALSE FALSE FALSE +1001 TriciaG No No No FALSE FALSE FALSE +804 FirstKnight Yes Yes No TRUE FALSE TRUE +1641 Kirsten Wever No No No FALSE FALSE FALSE +1259 Megan Argo No No No FALSE FALSE FALSE +1231 Abigail Bartels No No No FALSE FALSE FALSE +1410 Zachary Johnson Yes No No TRUE FALSE FALSE +1030 Ancient mariner Yes No No TRUE FALSE FALSE +1093 Katie Riley No No No FALSE FALSE FALSE +1254 Rosie No No No FALSE FALSE FALSE +1365 Eric Leach Yes No No TRUE FALSE FALSE +831 David Federman No No No FALSE FALSE FALSE +1989 Joannemmp No No No FALSE FALSE FALSE +1707 David Olson No No No FALSE FALSE FALSE +1849 Fred DeBerardinis Yes No No TRUE FALSE FALSE +1808 Rebecca King Yes No Yes TRUE FALSE TRUE +2292 Arnold No No No FALSE FALSE FALSE +2415 Patrick Eaton No No No FALSE FALSE FALSE +1656 Sharon Omi Yes Yes No TRUE FALSE TRUE +1676 Gargoyle No Yes No TRUE FALSE FALSE +1881 Julienne No No Yes TRUE FALSE FALSE +2036 T.K. Kirven No No No FALSE FALSE FALSE +1761 EliMarieHK Yes No No TRUE FALSE FALSE +2115 Pete Milan Yes Yes Yes TRUE TRUE TRUE +1803 Susan Hanfield Yes No No TRUE FALSE FALSE +1798 C. L. W. Rollins Yes No No TRUE FALSE FALSE +1723 Rachel Bossier Yes No No TRUE FALSE FALSE +2341 Haili Yes No Yes TRUE FALSE TRUE +2468 Erin Schellhase Yes Yes No TRUE FALSE TRUE +1725 Ruth Kidson Yes No No TRUE FALSE FALSE +2010 Peggy Yes No No TRUE FALSE FALSE +1853 Ron Altman Yes No No TRUE FALSE FALSE +2359 Doug Reed No No No FALSE FALSE FALSE +2422 Jude Somers No No No FALSE FALSE FALSE +2234 Coreena No No No FALSE FALSE FALSE +2156 C F de Rosset No No Yes TRUE FALSE FALSE +2483 Tammy Porter No No No FALSE FALSE FALSE +1781 humanode No No No FALSE FALSE FALSE +2275 NatalieOram No No No FALSE FALSE FALSE +2390 sdaeley17 No No No FALSE FALSE FALSE +2314 Cheri Jordan No No No FALSE FALSE FALSE +2413 Joanne Rochon No No No FALSE FALSE FALSE +1697 Lonelle Yoder No No No FALSE FALSE FALSE +1718 Caroline Driggs Yes Yes No TRUE FALSE TRUE +2387 Brett G. Hirsch No No No FALSE FALSE FALSE +2331 Madam Fickle No No No FALSE FALSE FALSE +1783 Sarah Crampton Yes Yes Yes TRUE TRUE TRUE +2397 Rebecca Braunert-Plunkett No No No FALSE FALSE FALSE +2357 William Gavula No No No FALSE FALSE FALSE +1670 dmbrought No No No FALSE FALSE FALSE +1987 Andrew White No No No FALSE FALSE FALSE +1755 Yvonne Smith Yes Yes No TRUE FALSE TRUE +2192 Sammy Bean Yes Yes No TRUE FALSE TRUE +1716 EyeBones Yes No No TRUE FALSE FALSE +1828 David Wales No No No FALSE FALSE FALSE +2251 Wiley Combs No No No FALSE FALSE FALSE +2065 Muriel No No No FALSE FALSE FALSE +2017 CaprishaPage Yes No No TRUE FALSE FALSE +1947 Barbara Edelman No No Yes TRUE FALSE FALSE +1738 Lois C. Johnson No No No FALSE FALSE FALSE +1791 David Cummings No Yes No TRUE FALSE FALSE +2045 Linda Ciano No No No FALSE FALSE FALSE +2452 Walt Allan No No Yes TRUE FALSE FALSE +2040 MJ Franck No No No FALSE FALSE FALSE +1831 Nigel Boydell Yes No No TRUE FALSE FALSE +2371 Alexander Hatton Yes Yes No TRUE FALSE TRUE +1954 Szindbad No No No FALSE FALSE FALSE +1836 Kendall Ashyby Yes Yes No TRUE FALSE TRUE +2436 josembi No No No FALSE FALSE FALSE +2383 Emma Joyce No No No FALSE FALSE FALSE +2278 Jake Woldstad No No No FALSE FALSE FALSE +1741 anjieliu Yes Yes No TRUE FALSE TRUE +1857 Amanda Friday Yes No No TRUE FALSE FALSE +2370 gloriousjob No No No FALSE FALSE FALSE +1907 Snapdragon No No No FALSE FALSE FALSE +2225 nomorejeffs No Yes No TRUE FALSE FALSE +2439 KHand Yes Yes No TRUE FALSE TRUE +2239 amaskill No Yes No TRUE FALSE FALSE +2007 Art Leung No No No FALSE FALSE FALSE +2283 Tim Cote No No No FALSE FALSE FALSE +1712 Steve Belleguelle Yes Yes Yes TRUE TRUE TRUE +2094 Meg Cowan Yes No No TRUE FALSE FALSE +1772 haggisreflux No No No FALSE FALSE FALSE +2317 helengraves Yes No No TRUE FALSE FALSE +2241 Steven Reynolds No No No FALSE FALSE FALSE +2011 pekein Yes No No TRUE FALSE FALSE +1826 John Hoerr Yes No No TRUE FALSE FALSE +1695 Tina Nuzzi Yes No No TRUE FALSE FALSE +2451 DeanOBuchanan Yes No No TRUE FALSE FALSE +1771 Chelsea S. Yes No No TRUE FALSE FALSE +2441 Alison Stewart No No No FALSE FALSE FALSE +1745 Janet No No No FALSE FALSE FALSE +2358 Betty Perry No No No FALSE FALSE FALSE +2197 Mike Nelson Yes Yes Yes TRUE TRUE TRUE +2014 Eden Rea-Hedrick Yes No No TRUE FALSE FALSE +1672 Mike Wajda No No No FALSE FALSE FALSE +2394 TinaNygard2 No Yes No TRUE FALSE FALSE +1657 alwpoe No No No FALSE FALSE FALSE +1728 Vinnie Tesla Yes Yes No TRUE FALSE TRUE +1805 Vince Dee Yes No Yes TRUE FALSE TRUE +2143 Suebee Yes Yes No TRUE FALSE TRUE +2084 Eberle Thomas Yes No Yes TRUE FALSE TRUE +2479 Daisy Flaim No No No FALSE FALSE FALSE +2152 Kristel Tretter No No No FALSE FALSE FALSE +2268 Greg Giordano No No No FALSE FALSE FALSE +1839 James E. Carson No No No FALSE FALSE FALSE +2056 acloward Yes Yes No TRUE FALSE TRUE +1814 polkadotish No No No FALSE FALSE FALSE +2127 Ron Lockhart No No No FALSE FALSE FALSE +2114 Larry Beasley No Yes Yes TRUE FALSE TRUE +2469 Kevin Owens Yes No No TRUE FALSE FALSE +2447 Deena Rhoads Yes Yes No TRUE FALSE TRUE +1724 Juliana M. Yes No No TRUE FALSE FALSE +1869 NastassiaS Yes Yes Yes TRUE TRUE TRUE +2209 Samantha J Gubitz Yes Yes Yes TRUE TRUE TRUE +2171 Carolyne No Yes No TRUE FALSE FALSE +2403 Ian Quinlan Yes No No TRUE FALSE FALSE +2032 doonaboon Yes No No TRUE FALSE FALSE +x Joy S Grape No No No FALSE FALSE FALSE \ No newline at end of file diff --git a/config/phone2idx.json b/config/phone2idx.json new file mode 100644 index 0000000000000000000000000000000000000000..c728de2ae024e01be46a86ad75774fc916638694 --- /dev/null +++ b/config/phone2idx.json @@ -0,0 +1,65 @@ +{ + "": 1, + "": 0, + "": 2, + "[COMMA]": 17, + "[EXCLAMATION MARK]": 50, + "[FULL STOP]": 30, + "[QUESTION MARK]": 54, + "[SEMICOLON]": 56, + "[SILENCE]": 3, + "a\u026a": 26, + "a\u028a": 46, + "b": 21, + "d": 19, + "d\u0292": 42, + "e\u026a": 4, + "f": 27, + "h": 38, + "i": 8, + "j": 31, + "k": 23, + "l": 22, + "m": 10, + "n": 15, + "o\u028a": 32, + "p": 11, + "s": 28, + "t": 13, + "t\u0283": 43, + "u": 24, + "v": 5, + "w": 29, + "z": 20, + "\u00e6": 18, + "\u00f0": 34, + "\u014b": 36, + "\u0251": 39, + "\u0254": 12, + "\u0254\u026a": 16, + "\u025b": 6, + "\u025c\u02de": 37, + "\u0261": 33, + "\u026a": 9, + "\u0279": 7, + "\u0283": 35, + "\u028a": 49, + "\u028c": 14, + "\u0292": 25, + "\u02cca\u026a": 52, + "\u02cca\u028a": 60, + "\u02cce\u026a": 58, + "\u02cci": 44, + "\u02cco\u028a": 45, + "\u02ccu": 51, + "\u02cc\u00e6": 59, + "\u02cc\u0251": 61, + "\u02cc\u0254": 57, + "\u02cc\u0254\u026a": 62, + "\u02cc\u025b": 53, + "\u02cc\u025c\u02de": 47, + "\u02cc\u026a": 40, + "\u02cc\u028a": 48, + "\u02cc\u028c": 55, + "\u03b8": 41 +} \ No newline at end of file diff --git a/config/speaker2idx.json b/config/speaker2idx.json new file mode 100644 index 0000000000000000000000000000000000000000..84a5e7ead40ff2bd4aab0678d97b07e596d01268 --- /dev/null +++ b/config/speaker2idx.json @@ -0,0 +1,2442 @@ +{ + "100": 1017, + "1006": 1635, + "101": 718, + "1012": 706, + "1018": 688, + "102": 2191, + "1025": 588, + "1027": 591, + "1028": 291, + "103": 15, + "1034": 182, + "104": 1979, + "1040": 195, + "1046": 459, + "1049": 2264, + "1050": 819, + "1051": 2174, + "1052": 374, + "1053": 878, + "1054": 864, + "1060": 1001, + "1061": 576, + "1065": 1420, + "1066": 366, + "1069": 210, + "107": 1333, + "1079": 381, + "1081": 190, + "1084": 1514, + "1085": 1288, + "1088": 107, + "1089": 2383, + "1092": 1555, + "1093": 480, + "1094": 1588, + "1096": 1339, + "1097": 1311, + "1098": 92, + "110": 1506, + "1100": 1080, + "1107": 1355, + "111": 1659, + "1110": 1540, + "1112": 489, + "1116": 54, + "112": 560, + "1121": 755, + "1124": 2160, + "1132": 1837, + "114": 1044, + "115": 1088, + "1152": 1417, + "1154": 1936, + "116": 2354, + "1160": 252, + "1161": 1405, + "1165": 694, + "1166": 2026, + "1168": 1516, + "1171": 1940, + "1179": 1547, + "118": 235, + "1182": 621, + "1183": 188, + "1184": 2107, + "1187": 1648, + "1188": 2404, + "119": 435, + "1195": 774, + "1200": 1382, + "121": 2384, + "1212": 1018, + "122": 490, + "1221": 2401, + "1222": 520, + "1224": 545, + "1225": 2065, + "1226": 396, + "123": 1319, + "1230": 1682, + "1235": 212, + "1239": 1670, + "1241": 991, + "1246": 135, + "125": 97, + "1250": 2037, + "1252": 1488, + "1255": 2342, + "1258": 2222, + "1259": 286, + "126": 712, + "1260": 1808, + "1261": 2275, + "1263": 3, + "1264": 1035, + "1265": 660, + "1266": 1235, + "127": 1182, + "1271": 812, + "1272": 2324, + "1274": 1287, + "128": 1343, + "1280": 1774, + "1283": 658, + "1284": 2377, + "1289": 1099, + "1290": 1033, + "1291": 2002, + "1296": 699, + "1298": 1560, + "1311": 734, + "1313": 323, + "1316": 1113, + "1320": 2390, + "1322": 890, + "1323": 365, + "133": 2129, + "1331": 1811, + "1334": 49, + "1335": 427, + "1336": 818, + "1337": 865, + "1341": 2082, + "1342": 1232, + "1343": 811, + "1347": 2070, + "1348": 849, + "1349": 501, + "1353": 1361, + "1355": 242, + "1363": 147, + "1365": 722, + "1367": 1415, + "1370": 1384, + "1373": 2018, + "1374": 2009, + "1379": 728, + "1382": 1066, + "1383": 1032, + "1384": 1300, + "1387": 580, + "1390": 1056, + "1392": 391, + "14": 1090, + "1401": 967, + "1403": 1735, + "1413": 894, + "1414": 1806, + "1417": 565, + "1421": 1545, + "1422": 461, + "1425": 806, + "1430": 1718, + "1444": 1934, + "1445": 413, + "1446": 907, + "1447": 130, + "1448": 333, + "1455": 216, + "1460": 715, + "1462": 2298, + "1463": 462, + "1469": 1144, + "147": 2180, + "1472": 1045, + "1473": 962, + "1474": 1308, + "1482": 1089, + "1485": 1589, + "1487": 277, + "1492": 1981, + "1494": 1641, + "1495": 1403, + "1498": 340, + "150": 207, + "1502": 56, + "1505": 1145, + "1509": 248, + "1513": 371, + "152": 1457, + "153": 1702, + "1535": 502, + "1536": 437, + "154": 402, + "1544": 2039, + "1545": 1483, + "1547": 604, + "1552": 493, + "1553": 167, + "1556": 736, + "1559": 2027, + "1563": 2050, + "1564": 1503, + "1566": 1826, + "1569": 1796, + "157": 899, + "1571": 825, + "1572": 1884, + "1578": 36, + "1579": 1360, + "1580": 2387, + "1585": 2363, + "159": 777, + "1593": 1631, + "1594": 52, + "1595": 1472, + "16": 534, + "1601": 1900, + "1603": 650, + "1607": 814, + "161": 1911, + "1614": 1777, + "1618": 1351, + "1621": 2086, + "1624": 38, + "1629": 980, + "163": 106, + "1630": 2365, + "1633": 1400, + "1636": 1548, + "1638": 884, + "1639": 341, + "1641": 257, + "1643": 1799, + "1645": 696, + "1646": 1667, + "1647": 1272, + "1648": 1684, + "1649": 929, + "1650": 2367, + "1651": 2341, + "1653": 1766, + "166": 1094, + "1664": 2153, + "1665": 1734, + "1668": 704, + "167": 2038, + "1673": 2313, + "1674": 1281, + "1678": 1125, + "1679": 1170, + "168": 1452, + "1680": 1717, + "1681": 1991, + "1685": 1204, + "1686": 2362, + "1688": 2432, + "1690": 1791, + "1691": 1871, + "1693": 1315, + "1695": 1908, + "1696": 1743, + "1699": 1500, + "17": 934, + "1701": 2345, + "1704": 1963, + "1705": 581, + "1708": 1499, + "1710": 2214, + "1714": 1539, + "1717": 1662, + "1721": 1328, + "1724": 933, + "1726": 2146, + "173": 1848, + "1731": 626, + "1733": 1931, + "1734": 867, + "1736": 1152, + "1737": 122, + "174": 2316, + "1740": 999, + "1743": 115, + "1746": 1410, + "1748": 361, + "175": 506, + "1750": 1525, + "1752": 846, + "1754": 529, + "1756": 2103, + "1757": 1825, + "176": 546, + "1760": 1285, + "1765": 1953, + "1767": 2020, + "1769": 799, + "177": 1926, + "1772": 1278, + "1773": 1397, + "1776": 904, + "1777": 563, + "1779": 403, + "1780": 1151, + "1784": 1517, + "1789": 445, + "1795": 1523, + "1801": 383, + "1804": 2063, + "1806": 926, + "1809": 1879, + "1811": 835, + "1813": 2220, + "1815": 1370, + "1819": 1439, + "1825": 264, + "1826": 772, + "1827": 1060, + "1828": 1805, + "1841": 81, + "1844": 1567, + "1845": 1110, + "1846": 1293, + "1849": 349, + "1851": 447, + "1859": 398, + "1863": 1496, + "1867": 78, + "1868": 1910, + "1870": 2029, + "1874": 372, + "1878": 1642, + "188": 679, + "1885": 615, + "1898": 153, + "19": 200, + "1901": 1873, + "1903": 465, + "1913": 515, + "1914": 1059, + "1919": 2314, + "192": 1013, + "1920": 2261, + "1923": 633, + "1924": 2081, + "1926": 133, + "1931": 1758, + "1933": 882, + "1938": 1990, + "1943": 412, + "1944": 375, + "1958": 913, + "196": 39, + "1961": 961, + "1963": 154, + "1968": 1449, + "1970": 20, + "1974": 289, + "1977": 2132, + "198": 65, + "1985": 2112, + "1987": 516, + "1988": 2325, + "1989": 1888, + "199": 1291, + "1992": 105, + "1993": 2320, + "1995": 2380, + "1998": 2434, + "20": 2145, + "200": 128, + "2001": 1719, + "2002": 120, + "2003": 1869, + "2004": 703, + "2007": 143, + "201": 124, + "2010": 671, + "2012": 321, + "2013": 1299, + "202": 1336, + "2021": 1153, + "2026": 2116, + "203": 555, + "2033": 2410, + "2035": 2335, + "204": 629, + "2042": 1951, + "2045": 1137, + "2046": 1528, + "205": 473, + "2050": 1264, + "2051": 1703, + "2053": 863, + "2056": 1046, + "2060": 355, + "2061": 397, + "2062": 1592, + "2063": 1691, + "2067": 1737, + "2068": 1373, + "207": 306, + "2074": 1079, + "2078": 2322, + "208": 550, + "2086": 2301, + "2089": 1894, + "209": 460, + "2090": 1380, + "2092": 67, + "2093": 1116, + "2096": 1332, + "210": 376, + "2100": 1950, + "2104": 1526, + "211": 234, + "2110": 702, + "2113": 441, + "2122": 1292, + "2127": 564, + "2133": 1507, + "2136": 70, + "2137": 946, + "2140": 1185, + "2143": 1720, + "2146": 930, + "2148": 2167, + "2149": 761, + "215": 1664, + "2152": 1842, + "2156": 385, + "2159": 151, + "216": 950, + "2162": 1023, + "2167": 610, + "217": 969, + "218": 1818, + "2182": 136, + "2185": 1577, + "2194": 820, + "2195": 1852, + "2196": 44, + "2198": 1176, + "22": 1036, + "2201": 853, + "2204": 1130, + "2208": 2157, + "2229": 271, + "2230": 451, + "2234": 1882, + "2237": 1663, + "2238": 478, + "224": 386, + "2240": 282, + "2246": 1501, + "225": 549, + "2254": 318, + "2256": 840, + "226": 171, + "2262": 1316, + "2269": 443, + "227": 487, + "2270": 1222, + "2272": 247, + "2273": 1167, + "2275": 1922, + "2276": 1263, + "2277": 2318, + "2279": 2204, + "228": 1974, + "2284": 1344, + "2285": 931, + "2288": 2139, + "2289": 99, + "229": 95, + "2292": 2236, + "2294": 775, + "2297": 1678, + "2299": 690, + "2300": 2371, + "2301": 2102, + "2309": 2195, + "231": 698, + "2312": 1538, + "2319": 705, + "233": 232, + "2339": 1581, + "2341": 1909, + "2346": 2055, + "2351": 2189, + "2356": 2284, + "2361": 1537, + "2364": 296, + "2368": 255, + "237": 2394, + "2374": 2005, + "238": 2295, + "2380": 1794, + "2384": 90, + "2388": 940, + "2391": 184, + "2393": 611, + "2397": 841, + "240": 648, + "2401": 787, + "2404": 627, + "2405": 1513, + "2407": 1983, + "2411": 914, + "2412": 2329, + "2414": 2435, + "2416": 42, + "242": 614, + "2427": 876, + "2428": 2332, + "243": 1455, + "2436": 215, + "2437": 1699, + "2445": 2229, + "2448": 1622, + "245": 1815, + "246": 1004, + "2473": 888, + "248": 61, + "2481": 975, + "2485": 1470, + "2487": 1711, + "2488": 1790, + "249": 488, + "2491": 1729, + "2494": 517, + "2496": 2289, + "2498": 322, + "2499": 297, + "25": 2098, + "250": 214, + "2504": 2017, + "2506": 2338, + "251": 2300, + "2512": 419, + "2514": 246, + "2517": 721, + "2518": 2, + "252": 1421, + "2522": 2034, + "2526": 2054, + "253": 1377, + "2531": 431, + "2532": 997, + "254": 244, + "2541": 2119, + "2544": 2022, + "2545": 1716, + "255": 1236, + "2552": 1409, + "2553": 1792, + "2562": 938, + "2568": 1585, + "2570": 693, + "2573": 628, + "2574": 1363, + "2577": 622, + "258": 334, + "2581": 1050, + "2582": 1133, + "2587": 1143, + "2588": 1753, + "2589": 836, + "2592": 911, + "2598": 251, + "26": 66, + "260": 2369, + "2606": 2003, + "2607": 1210, + "2609": 2437, + "2618": 1078, + "2624": 1372, + "2628": 345, + "263": 1732, + "2638": 434, + "264": 2105, + "265": 2059, + "2652": 307, + "2654": 317, + "2660": 1690, + "2671": 2109, + "2673": 476, + "2674": 384, + "2676": 1401, + "2688": 996, + "2691": 76, + "2694": 1265, + "27": 155, + "2709": 1009, + "2712": 1645, + "272": 763, + "2724": 1654, + "273": 1462, + "2730": 2232, + "2733": 2200, + "2735": 2183, + "274": 508, + "2740": 2260, + "2741": 596, + "2748": 2252, + "2751": 1051, + "2754": 1741, + "2758": 881, + "2762": 2130, + "2764": 104, + "2769": 766, + "277": 1787, + "2774": 1070, + "2775": 409, + "278": 639, + "2785": 793, + "2787": 665, + "2790": 901, + "28": 503, + "2803": 2321, + "2812": 429, + "2817": 103, + "2823": 1031, + "2825": 1921, + "2827": 758, + "283": 1639, + "2830": 2405, + "2834": 1201, + "2836": 10, + "2843": 233, + "2853": 309, + "2854": 1722, + "288": 912, + "2882": 778, + "289": 12, + "2893": 86, + "2895": 1156, + "29": 2006, + "2902": 2296, + "2909": 1913, + "2910": 30, + "2911": 239, + "2919": 1608, + "2920": 582, + "2925": 1367, + "2929": 1122, + "2930": 1846, + "294": 1224, + "2943": 1630, + "2946": 2242, + "2952": 198, + "296": 916, + "2960": 353, + "2961": 2381, + "2967": 1390, + "2971": 253, + "2975": 2288, + "2979": 1851, + "298": 101, + "2985": 2101, + "2988": 1595, + "2989": 114, + "2990": 1948, + "2992": 707, + "2997": 1456, + "2998": 2048, + "2999": 740, + "30": 426, + "3000": 2305, + "3001": 1042, + "3003": 477, + "3005": 2429, + "3006": 1668, + "3008": 771, + "3009": 857, + "302": 209, + "3020": 2270, + "3021": 2172, + "3025": 654, + "303": 513, + "3032": 891, + "3033": 1723, + "3045": 1864, + "3046": 692, + "3053": 1190, + "3054": 1280, + "3060": 1554, + "3063": 1274, + "307": 134, + "3070": 1041, + "3072": 1016, + "3079": 1985, + "3080": 2423, + "3081": 2299, + "3082": 482, + "3083": 1103, + "3088": 1930, + "3090": 1441, + "3094": 292, + "3097": 1971, + "3098": 1801, + "31": 1168, + "310": 1524, + "3100": 1977, + "3105": 414, + "3109": 1282, + "311": 74, + "3112": 71, + "3114": 261, + "3118": 484, + "3119": 573, + "3125": 1958, + "313": 1165, + "3132": 2030, + "3135": 2274, + "3137": 1640, + "3138": 2033, + "3142": 1219, + "3143": 2056, + "3144": 2087, + "3148": 1789, + "3157": 764, + "3168": 8, + "317": 2278, + "3170": 2311, + "3171": 1025, + "3172": 2249, + "3179": 1875, + "318": 1054, + "3180": 592, + "3185": 951, + "3187": 1112, + "319": 2123, + "3192": 1325, + "3196": 1141, + "32": 236, + "3214": 173, + "3215": 657, + "322": 32, + "3221": 943, + "3224": 360, + "3227": 2206, + "3228": 956, + "323": 877, + "3230": 497, + "3235": 144, + "3238": 2150, + "3240": 196, + "3242": 157, + "3244": 2186, + "3245": 1189, + "3257": 1478, + "3258": 324, + "3259": 64, + "3261": 1426, + "3268": 1681, + "3271": 2053, + "3272": 1378, + "3274": 708, + "3285": 2067, + "3288": 2162, + "3289": 710, + "329": 1105, + "3290": 2043, + "3294": 1029, + "3307": 354, + "331": 2007, + "3314": 1653, + "3318": 1923, + "3319": 1530, + "332": 192, + "3328": 928, + "3330": 880, + "3331": 2421, + "3334": 1952, + "3340": 663, + "3346": 1563, + "335": 606, + "3356": 2092, + "3357": 666, + "336": 2135, + "3361": 467, + "3368": 870, + "337": 574, + "3370": 695, + "3373": 1463, + "3374": 75, + "3379": 406, + "3380": 454, + "3381": 1757, + "3389": 358, + "339": 843, + "3394": 1809, + "340": 393, + "3400": 1998, + "3409": 1171, + "3411": 1284, + "3417": 1180, + "3433": 1413, + "3436": 93, + "3440": 159, + "3446": 336, + "3448": 587, + "345": 408, + "3465": 1708, + "3467": 1686, + "3470": 2169, + "3479": 1740, + "348": 1600, + "3482": 280, + "3483": 854, + "3486": 223, + "3488": 2036, + "3490": 326, + "3493": 917, + "3500": 1419, + "3503": 1713, + "3513": 430, + "3521": 369, + "3526": 29, + "3528": 2413, + "353": 701, + "3536": 2297, + "3537": 320, + "3538": 2424, + "3540": 892, + "3541": 1479, + "3546": 608, + "3547": 1194, + "3549": 869, + "3551": 616, + "3553": 1890, + "3554": 1468, + "3557": 1350, + "3559": 1752, + "3564": 1223, + "3567": 1984, + "3570": 2373, + "3571": 1629, + "3575": 2370, + "3576": 2331, + "3584": 700, + "3587": 1725, + "3588": 1804, + "359": 982, + "3592": 1853, + "3595": 2155, + "3598": 2207, + "36": 1557, + "3606": 1982, + "3607": 73, + "3615": 848, + "3618": 1329, + "362": 267, + "3630": 494, + "3638": 714, + "3641": 1392, + "3645": 536, + "3647": 2110, + "365": 1533, + "3650": 1249, + "3654": 404, + "3656": 2061, + "3657": 2271, + "366": 1989, + "3660": 2346, + "3663": 2361, + "3664": 53, + "3665": 1784, + "367": 2431, + "3675": 2120, + "3679": 1823, + "3681": 1313, + "3686": 883, + "369": 682, + "3691": 1460, + "3698": 1795, + "3699": 46, + "37": 1996, + "3703": 523, + "3717": 900, + "3723": 161, + "3728": 357, + "3729": 2388, + "373": 781, + "3733": 535, + "3738": 674, + "374": 57, + "3744": 1267, + "3747": 2118, + "3752": 2326, + "3757": 2171, + "3764": 2414, + "377": 1317, + "3779": 1986, + "3780": 1191, + "3781": 613, + "3783": 1221, + "3790": 949, + "3792": 988, + "3793": 1978, + "3796": 2138, + "3798": 2156, + "38": 263, + "380": 584, + "3807": 185, + "3816": 858, + "3819": 1440, + "3825": 1055, + "3830": 123, + "3835": 831, + "3843": 1896, + "3845": 1146, + "3848": 2151, + "3851": 382, + "3852": 749, + "3853": 2306, + "3857": 194, + "3864": 273, + "3866": 533, + "3867": 1997, + "3869": 717, + "3871": 1955, + "3876": 496, + "3879": 60, + "3885": 1677, + "3889": 875, + "3894": 1927, + "3895": 1833, + "3896": 1939, + "39": 118, + "3905": 464, + "3906": 1279, + "3909": 2010, + "3911": 1163, + "3912": 1237, + "3914": 1097, + "3915": 2356, + "392": 2165, + "3922": 578, + "3923": 833, + "3925": 1904, + "3926": 2163, + "3927": 1052, + "3928": 1572, + "3934": 1254, + "3945": 810, + "3947": 17, + "3955": 1814, + "3959": 1824, + "3962": 2243, + "3967": 995, + "3969": 1807, + "3972": 313, + "3977": 298, + "3979": 509, + "398": 725, + "3982": 13, + "3983": 16, + "3989": 1076, + "3990": 1957, + "3992": 1812, + "3994": 519, + "3997": 2416, + "40": 146, + "4005": 1933, + "4009": 1883, + "4010": 387, + "4013": 599, + "4014": 186, + "4015": 2136, + "4017": 2246, + "4018": 229, + "4019": 1529, + "402": 2008, + "4020": 2216, + "4021": 1340, + "403": 68, + "4034": 1511, + "4039": 301, + "404": 1348, + "4042": 1969, + "4044": 567, + "405": 165, + "4051": 110, + "4054": 813, + "4057": 680, + "4059": 2104, + "4063": 1314, + "4064": 669, + "4077": 2374, + "4078": 1532, + "408": 294, + "4085": 2281, + "4088": 100, + "409": 401, + "4090": 1465, + "4098": 595, + "4104": 2244, + "4108": 797, + "4110": 741, + "4111": 356, + "4116": 893, + "412": 82, + "4122": 1212, + "413": 1251, + "4133": 942, + "4137": 193, + "4138": 344, + "4145": 780, + "4148": 1049, + "4152": 711, + "4153": 2344, + "4156": 2077, + "4160": 142, + "4161": 1575, + "4172": 1854, + "4174": 2078, + "4179": 1320, + "4189": 1660, + "4191": 1858, + "4192": 1756, + "4193": 1973, + "4195": 111, + "4196": 1423, + "4198": 2418, + "4205": 2256, + "421": 1389, + "4211": 2089, + "4214": 206, + "4216": 1899, + "4217": 1173, + "4218": 1381, + "422": 2327, + "4222": 919, + "4225": 1938, + "4226": 958, + "4234": 1889, + "4235": 1566, + "4236": 927, + "4238": 759, + "4243": 279, + "4246": 1100, + "4257": 388, + "426": 160, + "4260": 1015, + "4262": 1785, + "4263": 1215, + "4267": 222, + "4273": 2013, + "4277": 1840, + "4278": 1003, + "428": 1775, + "4280": 1230, + "4289": 993, + "4290": 885, + "4294": 2422, + "4295": 1721, + "4297": 112, + "4305": 1166, + "4310": 1672, + "4313": 1679, + "432": 2224, + "4321": 1353, + "4323": 2353, + "4327": 1891, + "4331": 265, + "4335": 689, + "4340": 183, + "4344": 1607, + "4345": 1368, + "4350": 2430, + "4352": 1779, + "4356": 314, + "4358": 924, + "4362": 219, + "4363": 619, + "4379": 1887, + "4381": 752, + "439": 742, + "4396": 1228, + "4397": 132, + "44": 2175, + "4402": 1599, + "4406": 231, + "4407": 1327, + "4411": 1365, + "4415": 1583, + "4420": 2060, + "4422": 1932, + "4423": 2088, + "4425": 968, + "4427": 303, + "4428": 1627, + "4434": 649, + "4438": 1117, + "444": 1769, + "4441": 224, + "4442": 1508, + "4443": 1847, + "4446": 2392, + "4447": 2255, + "4455": 1341, + "446": 178, + "4463": 1688, + "4474": 1399, + "448": 2122, + "4480": 1820, + "4481": 170, + "4484": 1406, + "4487": 1886, + "4490": 583, + "4492": 1243, + "4495": 618, + "45": 2011, + "4507": 2403, + "451": 652, + "4511": 1658, + "4513": 1632, + "4515": 2357, + "4519": 538, + "4520": 2040, + "453": 1535, + "4535": 603, + "454": 466, + "4545": 1418, + "4546": 1446, + "4549": 1771, + "4563": 1582, + "4570": 2358, + "4572": 2364, + "4576": 1411, + "458": 125, + "4583": 1793, + "4586": 274, + "459": 329, + "4590": 1064, + "4591": 2094, + "4592": 281, + "4594": 1302, + "4595": 1084, + "4598": 602, + "4599": 1917, + "46": 1707, + "460": 169, + "4629": 1019, + "4640": 129, + "4652": 1762, + "4659": 2205, + "466": 2286, + "4660": 1623, + "4667": 2192, + "4680": 148, + "4687": 1187, + "4693": 1486, + "4697": 1800, + "4699": 2108, + "47": 1733, + "470": 1676, + "4701": 2254, + "4703": 1261, + "4705": 1614, + "4706": 1786, + "4710": 1509, + "4712": 2001, + "4719": 791, + "472": 337, + "4731": 370, + "4733": 844, + "4734": 1000, + "4738": 1988, + "474": 2028, + "4741": 1593, + "4742": 1636, + "4744": 838, + "4748": 1915, + "475": 328, + "4750": 2062, + "4757": 1466, + "476": 278, + "4766": 1912, + "4767": 1863, + "4771": 1142, + "4773": 1745, + "4779": 1226, + "4788": 89, + "479": 491, + "4791": 2161, + "4799": 1482, + "480": 566, + "4800": 782, + "4806": 394, + "4807": 944, + "481": 37, + "4813": 174, + "4821": 1437, + "4824": 1175, + "483": 1502, + "4830": 18, + "4831": 2352, + "4836": 1767, + "4837": 609, + "4839": 992, + "4841": 1995, + "4846": 290, + "4848": 830, + "4852": 2420, + "4853": 48, + "4854": 486, + "4856": 971, + "4859": 131, + "4860": 379, + "4863": 1906, + "487": 1119, + "4872": 1695, + "489": 1149, + "4894": 1195, + "4898": 116, + "4899": 364, + "49": 1943, + "4910": 1918, + "4915": 1885, + "492": 1038, + "4926": 768, + "4930": 1257, + "4931": 1968, + "4936": 1902, + "4945": 1043, + "4948": 2090, + "4955": 1186, + "4957": 1068, + "4958": 2198, + "4959": 1644, + "4964": 1520, + "4965": 1960, + "4967": 1002, + "4969": 1647, + "497": 637, + "4970": 2375, + "4973": 823, + "4979": 1866, + "4991": 1233, + "4992": 2382, + "4993": 2000, + "500": 974, + "5000": 1174, + "5002": 620, + "5005": 1970, + "5007": 288, + "5009": 1751, + "501": 965, + "5012": 973, + "5013": 1297, + "5019": 1736, + "5022": 245, + "5023": 1469, + "5029": 817, + "5036": 1867, + "5038": 1693, + "5039": 547, + "5043": 1356, + "5044": 1665, + "5045": 1436, + "5049": 189, + "505": 1239, + "5054": 788, + "5060": 1290, + "5062": 532, + "5063": 597, + "5076": 1573, + "5077": 1746, + "5082": 2225, + "5092": 348, + "5093": 285, + "51": 1338, + "510": 939, + "5101": 1865, + "5104": 181, + "5105": 2406, + "511": 1128, + "5115": 352, + "5118": 1876, + "512": 955, + "5123": 589, + "5126": 977, + "5132": 2093, + "5133": 636, + "5136": 1772, + "5139": 1106, + "5141": 2181, + "5142": 2396, + "5147": 498, + "5152": 1471, + "5154": 450, + "5157": 659, + "5163": 77, + "5164": 1578, + "517": 2069, + "5172": 1450, + "5181": 2259, + "5183": 1803, + "5185": 1739, + "5186": 458, + "5189": 738, + "5190": 504, + "5192": 208, + "5198": 1928, + "5199": 2111, + "52": 2233, + "5206": 1065, + "5217": 1527, + "5220": 1443, + "5224": 1306, + "5230": 1712, + "5233": 1217, + "5239": 410, + "5242": 600, + "5244": 2097, + "5245": 1868, + "5246": 651, + "5248": 1543, + "5252": 2240, + "5261": 363, + "5266": 850, + "5269": 1568, + "5271": 2114, + "5278": 2154, + "5280": 1738, + "5285": 1956, + "5287": 2228, + "5290": 724, + "5293": 415, + "5296": 1694, + "5299": 2066, + "5304": 719, + "5319": 1124, + "5321": 2058, + "5322": 28, + "5325": 1916, + "5328": 1698, + "533": 2438, + "5333": 524, + "5337": 266, + "5338": 2310, + "5339": 94, + "534": 522, + "5340": 2291, + "5350": 2047, + "5355": 1225, + "5361": 1521, + "5375": 1844, + "5379": 1429, + "5386": 612, + "5389": 807, + "5390": 50, + "5393": 163, + "54": 979, + "5400": 631, + "5401": 577, + "5405": 1495, + "5412": 1542, + "542": 2227, + "5424": 2064, + "5429": 1687, + "543": 1135, + "5439": 2012, + "544": 1330, + "5442": 2411, + "5445": 1248, + "5448": 1040, + "5456": 14, + "5459": 2213, + "5460": 1907, + "5463": 150, + "5468": 1576, + "5471": 1451, + "548": 842, + "5480": 1454, + "5484": 2408, + "5487": 1643, + "5489": 1024, + "549": 970, + "55": 425, + "5506": 2083, + "551": 1208, + "5513": 667, + "5514": 121, + "5519": 1115, + "5536": 2328, + "5538": 792, + "5543": 2337, + "5545": 2100, + "5561": 179, + "5565": 1510, + "5567": 1947, + "5569": 1710, + "557": 1240, + "5570": 936, + "5583": 568, + "5588": 845, + "559": 1028, + "56": 531, + "5604": 346, + "5606": 751, + "561": 902, + "5618": 964, + "5620": 1813, + "5622": 1062, + "5628": 1200, + "5635": 675, + "5636": 1768, + "5639": 2391, + "5641": 2237, + "5649": 1250, + "5652": 221, + "5653": 1178, + "5655": 1109, + "5656": 986, + "5660": 530, + "5661": 1494, + "5665": 1778, + "567": 1750, + "5671": 1673, + "5672": 941, + "5678": 238, + "568": 1604, + "5682": 1697, + "5683": 2400, + "5684": 452, + "569": 1675, + "5694": 2334, + "57": 2283, + "5700": 2277, + "5703": 176, + "5712": 871, + "5717": 638, + "5719": 1179, + "572": 1550, + "5720": 1198, + "5723": 697, + "5724": 605, + "5725": 1685, + "5727": 754, + "5731": 1118, + "5733": 1202, + "5735": 1255, + "5740": 832, + "5746": 790, + "5750": 21, + "5756": 1628, + "576": 416, + "5764": 2417, + "5765": 2127, + "5767": 332, + "5772": 1616, + "5776": 553, + "5778": 168, + "5781": 2019, + "5784": 2203, + "5789": 22, + "5791": 1967, + "5796": 2131, + "58": 1359, + "580": 250, + "5802": 367, + "5808": 117, + "5809": 795, + "581": 684, + "5810": 572, + "5825": 2170, + "5826": 1193, + "583": 783, + "5831": 1657, + "5837": 1334, + "584": 1819, + "5840": 1562, + "5849": 2351, + "585": 1905, + "5854": 1782, + "5860": 1522, + "5867": 187, + "5868": 966, + "587": 109, + "5874": 1374, + "5876": 428, + "5883": 1014, + "5886": 1749, + "589": 829, + "5890": 2280, + "5893": 1558, + "5894": 1214, + "5895": 2307, + "5906": 1897, + "5909": 815, + "5910": 1949, + "5911": 1181, + "5913": 1312, + "5914": 632, + "5918": 925, + "5929": 1298, + "593": 733, + "5933": 1556, + "5935": 308, + "594": 527, + "5940": 541, + "5949": 1485, + "5951": 1310, + "5952": 2046, + "596": 947, + "5968": 798, + "597": 770, + "5970": 1870, + "5975": 254, + "5977": 2134, + "5979": 2095, + "598": 727, + "5980": 1937, + "5983": 1655, + "5984": 909, + "5985": 1008, + "5993": 1559, + "60": 108, + "6000": 11, + "6006": 753, + "6009": 1901, + "6010": 1689, + "6014": 745, + "6019": 140, + "6025": 2190, + "6030": 1726, + "6032": 1111, + "6035": 2208, + "6037": 1092, + "6038": 898, + "6051": 2164, + "6054": 855, + "606": 983, + "6060": 470, + "6064": 226, + "6065": 1386, + "6070": 2433, + "6072": 2219, + "6075": 1026, + "6076": 2035, + "6077": 2272, + "6078": 175, + "608": 1925, + "6080": 735, + "6081": 225, + "6082": 1071, + "6084": 1364, + "6087": 2265, + "6088": 1601, + "6097": 1724, + "6098": 284, + "6099": 731, + "6102": 1414, + "6104": 1022, + "6106": 1427, + "6111": 2215, + "6115": 1034, + "6119": 559, + "612": 1011, + "6120": 579, + "6121": 1383, + "6123": 2366, + "6126": 1929, + "6127": 1705, + "6128": 2412, + "613": 2223, + "6131": 1394, + "6135": 1744, + "6138": 1296, + "6139": 802, + "614": 1241, + "6145": 1855, + "6147": 172, + "6153": 1490, + "6157": 908, + "6159": 1388, + "6160": 987, + "6167": 1063, + "6173": 1477, + "6177": 1422, + "6178": 1881, + "6181": 119, + "6184": 2197, + "6188": 801, + "6189": 1086, + "6196": 1375, + "6199": 2021, + "62": 1262, + "6206": 923, + "6209": 149, + "6211": 1633, + "6215": 1077, + "622": 1337, + "6221": 1975, + "6224": 2211, + "6227": 2044, + "6232": 1242, + "6233": 683, + "6235": 359, + "6241": 2303, + "6242": 2125, + "6248": 1829, + "6249": 2226, + "625": 27, + "6251": 1481, + "6254": 2042, + "6258": 1027, + "6267": 2347, + "6269": 1101, + "6272": 26, + "6276": 1162, + "6281": 1570, + "6284": 1461, + "6286": 449, + "6288": 590, + "6294": 472, + "6295": 2308, + "6300": 319, + "6308": 514, + "6311": 1211, + "6313": 2304, + "6317": 351, + "6319": 2323, + "6323": 2245, + "6324": 1331, + "6330": 417, + "6332": 1586, + "6333": 2262, + "6339": 746, + "6341": 963, + "6345": 2319, + "6346": 1371, + "6351": 1651, + "6352": 518, + "6353": 1183, + "6356": 1318, + "6358": 1475, + "6359": 1053, + "636": 325, + "6364": 1728, + "6367": 6, + "6368": 2144, + "637": 776, + "6370": 2279, + "6371": 998, + "6373": 1085, + "6377": 1650, + "6378": 485, + "6385": 88, + "6388": 1102, + "639": 598, + "6391": 2168, + "6395": 389, + "6399": 2124, + "64": 439, + "6402": 1534, + "6406": 670, + "6407": 1857, + "6411": 1624, + "6415": 23, + "6418": 1430, + "6426": 1030, + "6432": 2427, + "6436": 2115, + "6437": 24, + "644": 1518, + "6446": 260, + "6454": 9, + "6455": 2343, + "6458": 558, + "6459": 1273, + "6467": 2340, + "6476": 80, + "6482": 2121, + "6484": 1541, + "6488": 1301, + "6492": 400, + "6494": 392, + "6497": 978, + "6499": 380, + "65": 2247, + "6505": 762, + "6506": 2263, + "6509": 862, + "6510": 921, + "6512": 2212, + "6513": 1727, + "6518": 1067, + "6519": 1057, + "652": 2302, + "6529": 25, + "6531": 79, + "6533": 2173, + "6534": 1621, + "6535": 2140, + "6538": 796, + "6539": 1444, + "6540": 1275, + "6544": 1132, + "6548": 2158, + "6549": 1362, + "6550": 760, + "6553": 479, + "6555": 1121, + "6557": 1412, + "6563": 91, + "6567": 786, + "6568": 1980, + "6574": 395, + "6575": 343, + "6583": 1289, + "6590": 1335, + "6594": 1357, + "6599": 2350, + "66": 1434, + "6609": 1619, + "6610": 1797, + "6614": 2293, + "6620": 800, + "6625": 1467, + "6627": 1821, + "663": 548, + "6636": 1832, + "6637": 335, + "664": 483, + "6643": 521, + "6652": 1258, + "666": 903, + "6660": 2149, + "6668": 2147, + "667": 686, + "6670": 1489, + "6673": 757, + "6674": 1487, + "6676": 1425, + "6683": 937, + "6685": 1385, + "6686": 726, + "6687": 2014, + "6689": 1874, + "669": 4, + "6690": 510, + "6694": 827, + "6695": 1247, + "6696": 1091, + "6701": 676, + "6705": 2187, + "6707": 1850, + "6709": 2276, + "6713": 1649, + "672": 2379, + "6724": 1942, + "6726": 1964, + "6727": 623, + "6733": 2201, + "6735": 2268, + "6741": 1387, + "6743": 1765, + "6746": 1408, + "6747": 2239, + "6749": 1512, + "6752": 1259, + "6753": 1515, + "6754": 1781, + "6758": 2117, + "6763": 1134, + "6773": 1625, + "6777": 1148, + "6782": 847, + "6784": 2188, + "6788": 421, + "679": 2143, + "6792": 1324, + "6794": 2024, + "6798": 1498, + "6804": 2178, + "6807": 2126, + "681": 2199, + "6818": 227, + "6821": 1590, + "6828": 897, + "6829": 2399, + "6836": 138, + "684": 1692, + "6841": 2336, + "6846": 1271, + "6848": 158, + "6849": 1294, + "6853": 1398, + "6865": 668, + "6875": 2031, + "6877": 784, + "688": 868, + "6880": 96, + "6882": 1326, + "6883": 1755, + "6892": 1862, + "6895": 601, + "690": 1605, + "6902": 1860, + "6904": 953, + "6906": 1459, + "6912": 1484, + "6913": 1464, + "6914": 1783, + "6918": 310, + "6923": 2032, + "6924": 569, + "6925": 164, + "6927": 839, + "6930": 2385, + "6937": 1007, + "6938": 2425, + "6943": 1491, + "6945": 1140, + "6947": 2113, + "6950": 1309, + "6951": 1961, + "6954": 2269, + "6956": 959, + "696": 241, + "6962": 2016, + "6963": 1321, + "6965": 809, + "6967": 1715, + "6974": 1652, + "6978": 1919, + "698": 935, + "6981": 1072, + "699": 1069, + "6993": 874, + "70": 456, + "700": 2360, + "7000": 647, + "7001": 1342, + "7008": 1270, + "7009": 1216, + "7011": 444, + "7012": 1269, + "7018": 2439, + "7021": 2393, + "7026": 2076, + "7030": 737, + "7046": 1197, + "705": 1536, + "7051": 433, + "7055": 1754, + "7059": 145, + "7061": 554, + "7062": 1474, + "7065": 1395, + "7067": 203, + "7069": 804, + "707": 952, + "7073": 1987, + "7078": 127, + "7079": 1620, + "708": 803, + "7085": 1047, + "7090": 258, + "7092": 1492, + "7095": 617, + "7096": 1252, + "7097": 1346, + "7105": 2428, + "7107": 1161, + "711": 678, + "7113": 204, + "7117": 551, + "712": 1349, + "7120": 989, + "7121": 1433, + "7125": 1780, + "7126": 895, + "7127": 2372, + "7128": 709, + "713": 1849, + "7131": 2106, + "7134": 661, + "7135": 1164, + "7138": 2267, + "7139": 275, + "7140": 981, + "7143": 1160, + "7145": 1005, + "7147": 2025, + "7148": 84, + "7150": 1596, + "7155": 1546, + "716": 474, + "7169": 440, + "7170": 1402, + "7176": 2386, + "7177": 1260, + "7178": 102, + "718": 561, + "7188": 748, + "7189": 1431, + "7190": 1, + "7197": 2142, + "7198": 1432, + "7199": 2166, + "720": 1358, + "7205": 1203, + "7208": 1841, + "7215": 1480, + "7218": 2152, + "7220": 1834, + "7223": 2282, + "7226": 152, + "7228": 1893, + "7229": 687, + "7238": 2241, + "7239": 1898, + "724": 994, + "7240": 422, + "7241": 918, + "7242": 1322, + "7245": 453, + "7246": 1828, + "7247": 773, + "7250": 2250, + "7255": 1976, + "7258": 1126, + "726": 1661, + "7263": 1379, + "7264": 213, + "7265": 1730, + "727": 1972, + "7276": 948, + "7277": 1602, + "7278": 240, + "728": 1205, + "7285": 270, + "7286": 293, + "7294": 1131, + "7297": 972, + "7299": 2234, + "730": 137, + "7301": 1878, + "7302": 62, + "7307": 2177, + "731": 299, + "7312": 41, + "7313": 643, + "7314": 860, + "7315": 1213, + "7316": 1010, + "7318": 1082, + "7320": 1617, + "7326": 1150, + "7327": 1218, + "7331": 1493, + "7333": 2080, + "7335": 985, + "7337": 1613, + "7338": 2079, + "7339": 861, + "7342": 826, + "7346": 1747, + "7348": 1638, + "7354": 1920, + "7357": 1671, + "7360": 2179, + "7367": 69, + "737": 1845, + "7376": 1773, + "7383": 653, + "7384": 906, + "7387": 2258, + "7389": 1603, + "7391": 1549, + "7392": 2230, + "7395": 575, + "7398": 552, + "7402": 217, + "7408": 2159, + "7416": 729, + "742": 1962, + "7423": 1924, + "7424": 1445, + "7433": 1965, + "7434": 1021, + "7436": 1941, + "7437": 1037, + "7445": 837, + "7447": 113, + "7448": 1895, + "7460": 368, + "7463": 1238, + "7467": 1157, + "7475": 765, + "7478": 571, + "7480": 2231, + "7481": 1020, + "7484": 1075, + "7491": 2072, + "7492": 1544, + "7495": 828, + "7498": 390, + "75": 1872, + "7502": 2210, + "7505": 31, + "7507": 2292, + "7510": 2176, + "7511": 33, + "7512": 2202, + "7514": 1354, + "7515": 327, + "7517": 211, + "7518": 420, + "7520": 302, + "7522": 2182, + "7525": 730, + "753": 1305, + "7538": 500, + "7540": 438, + "7552": 2004, + "7553": 834, + "7555": 805, + "7556": 1246, + "7558": 312, + "7559": 1827, + "7561": 1139, + "7565": 1199, + "7569": 910, + "7584": 1209, + "7585": 1817, + "7594": 557, + "7597": 1903, + "7601": 2348, + "7603": 1245, + "7607": 1877, + "7608": 1286, + "7609": 1177, + "7618": 2235, + "7635": 59, + "764": 976, + "7640": 1169, + "7641": 2349, + "7644": 2015, + "7647": 645, + "7649": 2266, + "7654": 1945, + "7657": 350, + "766": 1748, + "7665": 1061, + "7672": 1473, + "7679": 2141, + "7683": 1531, + "7687": 1307, + "7688": 1048, + "7691": 1612, + "7697": 2368, + "7699": 1396, + "77": 2253, + "770": 512, + "7700": 1253, + "7702": 1561, + "7704": 373, + "7705": 1095, + "7708": 2194, + "7713": 1188, + "7717": 586, + "7720": 789, + "7729": 2376, + "7730": 1087, + "7732": 1107, + "7733": 664, + "7737": 1347, + "7739": 295, + "774": 1669, + "7746": 1229, + "7749": 1680, + "7752": 411, + "7754": 1074, + "7756": 2148, + "7762": 1770, + "7764": 1192, + "7766": 418, + "7769": 1760, + "777": 2333, + "7777": 405, + "778": 1519, + "7780": 34, + "7783": 1093, + "7786": 2085, + "7789": 316, + "779": 2287, + "7794": 218, + "7795": 1634, + "7796": 1701, + "78": 243, + "780": 1836, + "7800": 191, + "7802": 330, + "7809": 879, + "781": 691, + "7816": 681, + "7823": 2290, + "7825": 634, + "7826": 1574, + "7828": 794, + "783": 347, + "7832": 1081, + "7833": 446, + "7835": 1571, + "7837": 856, + "7839": 1505, + "7843": 1369, + "7848": 1448, + "7850": 2315, + "7859": 230, + "7867": 945, + "7868": 1127, + "7871": 1580, + "7874": 915, + "7879": 1438, + "7881": 1104, + "7883": 1839, + "7886": 1551, + "789": 1838, + "7892": 1553, + "7898": 2073, + "79": 872, + "7902": 2436, + "7909": 505, + "791": 1764, + "7910": 896, + "7912": 1822, + "792": 1458, + "7923": 1303, + "7925": 1206, + "7926": 1058, + "7932": 562, + "7933": 852, + "7938": 542, + "7939": 607, + "7942": 1564, + "7945": 672, + "7946": 2096, + "7949": 624, + "7956": 873, + "7957": 259, + "7959": 539, + "7962": 283, + "7967": 744, + "797": 1892, + "7975": 2407, + "7976": 2312, + "7981": 481, + "7982": 1006, + "7988": 2023, + "7991": 1123, + "7994": 851, + "7995": 957, + "7997": 1244, + "8005": 2057, + "8006": 507, + "8008": 287, + "8009": 1831, + "8011": 1129, + "8012": 1696, + "8014": 19, + "8015": 2257, + "8023": 2218, + "8028": 920, + "803": 543, + "8033": 1428, + "8040": 2133, + "8042": 1227, + "8044": 2285, + "8050": 1096, + "8051": 126, + "8057": 767, + "8058": 1352, + "806": 1136, + "8063": 162, + "8066": 338, + "807": 1606, + "8071": 1442, + "8072": 1584, + "8075": 769, + "8080": 821, + "8087": 1830, + "8088": 58, + "8095": 237, + "8097": 455, + "8098": 205, + "81": 272, + "810": 1646, + "8108": 197, + "811": 2074, + "8112": 1966, + "8113": 889, + "8118": 269, + "8119": 377, + "8123": 55, + "8131": 2415, + "8138": 471, + "8142": 859, + "8143": 1709, + "8148": 1666, + "815": 448, + "8152": 362, + "8156": 1714, + "816": 677, + "8163": 423, + "8164": 1992, + "8168": 1611, + "8169": 1416, + "8172": 1497, + "8173": 2339, + "8176": 331, + "8180": 1776, + "8183": 644, + "8188": 2419, + "8190": 511, + "8193": 249, + "8194": 262, + "8195": 713, + "8197": 2099, + "8199": 2185, + "82": 1861, + "8200": 1552, + "8208": 1407, + "8215": 1159, + "8222": 656, + "8224": 2402, + "8225": 305, + "8226": 45, + "8228": 641, + "8230": 2389, + "8238": 47, + "8240": 2041, + "8242": 1276, + "8245": 1391, + "8246": 1268, + "8250": 1944, + "8254": 2355, + "8259": 1618, + "826": 1476, + "8262": 1999, + "8266": 463, + "8272": 1196, + "8273": 2217, + "8280": 2426, + "8288": 2359, + "829": 655, + "8291": 2084, + "8295": 1404, + "8296": 1704, + "8297": 2330, + "83": 72, + "830": 304, + "8300": 739, + "8302": 1946, + "8307": 1683, + "831": 5, + "8312": 156, + "8316": 1674, + "8321": 2052, + "8322": 2238, + "8324": 177, + "8328": 1424, + "8329": 808, + "8334": 1731, + "8337": 1856, + "8346": 1295, + "8347": 407, + "835": 1114, + "8356": 2209, + "836": 339, + "8367": 2075, + "8382": 1266, + "8388": 315, + "8389": 1598, + "839": 220, + "8392": 2137, + "8394": 1880, + "8396": 673, + "84": 2309, + "8401": 716, + "8404": 1039, + "8410": 640, + "8413": 1231, + "8414": 2045, + "8415": 1447, + "8419": 98, + "8421": 685, + "8422": 1366, + "8424": 1155, + "8425": 199, + "8430": 1345, + "8432": 1798, + "844": 1504, + "8441": 1700, + "8443": 1158, + "8444": 1184, + "8445": 1763, + "8447": 2091, + "845": 1435, + "8455": 2378, + "8459": 300, + "846": 1843, + "8461": 2409, + "8463": 2395, + "8464": 1108, + "8465": 43, + "8466": 1954, + "8468": 180, + "8470": 1147, + "8474": 756, + "8476": 1393, + "8479": 922, + "8490": 399, + "8494": 528, + "8498": 785, + "8499": 1569, + "85": 1994, + "850": 625, + "8500": 1788, + "8506": 630, + "851": 2068, + "8527": 646, + "8531": 1597, + "8534": 642, + "8536": 1591, + "8543": 2294, + "8544": 1304, + "8545": 816, + "8555": 2398, + "8565": 1220, + "8573": 732, + "8575": 424, + "8576": 1453, + "8580": 83, + "8587": 1959, + "8590": 1323, + "8591": 432, + "8592": 537, + "8605": 905, + "8609": 51, + "8619": 1012, + "8625": 1256, + "8629": 139, + "8630": 228, + "8631": 1609, + "8632": 2128, + "8635": 436, + "8643": 990, + "8644": 1283, + "8664": 1172, + "8666": 1637, + "8671": 1742, + "8675": 2251, + "8677": 540, + "8678": 1207, + "868": 662, + "8684": 720, + "8687": 824, + "8699": 378, + "87": 35, + "8705": 268, + "8710": 2273, + "8713": 469, + "8718": 886, + "8722": 743, + "8725": 468, + "8742": 556, + "8747": 166, + "8753": 1706, + "8758": 526, + "876": 1993, + "8765": 1935, + "8770": 141, + "8771": 1083, + "8772": 442, + "8776": 593, + "8778": 1859, + "8786": 747, + "8791": 822, + "8797": 63, + "8799": 1579, + "8803": 1810, + "8808": 2051, + "882": 276, + "8820": 525, + "8824": 954, + "8825": 544, + "8838": 87, + "884": 1154, + "8842": 2317, + "8846": 1587, + "8848": 475, + "8855": 492, + "886": 1610, + "8867": 2184, + "887": 40, + "8875": 495, + "8879": 984, + "8887": 594, + "8897": 2221, + "89": 201, + "895": 2248, + "8975": 85, + "899": 1073, + "90": 570, + "9000": 1656, + "9022": 635, + "9023": 887, + "9026": 866, + "908": 2397, + "909": 7, + "91": 1376, + "911": 202, + "915": 1816, + "92": 1565, + "920": 932, + "921": 2193, + "922": 457, + "923": 2071, + "925": 723, + "927": 1835, + "93": 499, + "937": 1277, + "94": 1759, + "948": 1138, + "949": 585, + "951": 1761, + "953": 960, + "954": 1098, + "956": 1615, + "957": 256, + "960": 1234, + "964": 1914, + "968": 1120, + "969": 2196, + "976": 1626, + "978": 1802, + "979": 750, + "98": 342, + "982": 1594, + "984": 779, + "985": 2049, + "986": 311, + "": 0 +} \ No newline at end of file diff --git a/config/speaker_id_mapping_libri.json b/config/speaker_id_mapping_libri.json new file mode 100644 index 0000000000000000000000000000000000000000..6612f37d4e6f853d9178780233123ef0cdf2f9be --- /dev/null +++ b/config/speaker_id_mapping_libri.json @@ -0,0 +1 @@ +{"14": 0, "16": 1, "17": 2, "19": 3, "20": 4, "22": 5, "23": 6, "25": 7, "26": 8, "27": 9, "28": 10, "29": 11, "30": 12, "31": 13, "32": 14, "36": 15, "37": 16, "38": 17, "39": 18, "40": 19, "44": 20, "45": 21, "46": 22, "47": 23, "49": 24, "51": 25, "52": 26, "54": 27, "55": 28, "56": 29, "57": 30, "58": 31, "60": 32, "61": 33, "62": 34, "64": 35, "65": 36, "66": 37, "70": 38, "75": 39, "77": 40, "78": 41, "79": 42, "81": 43, "82": 44, "83": 45, "84": 46, "85": 47, "87": 48, "89": 49, "90": 50, "91": 51, "92": 52, "93": 53, "94": 54, "98": 55, "100": 56, "101": 57, "102": 58, "103": 59, "104": 60, "107": 61, "110": 62, "111": 63, "112": 64, "114": 65, "115": 66, "116": 67, "118": 68, "119": 69, "121": 70, "122": 71, "123": 72, "125": 73, "126": 74, "127": 75, "128": 76, "133": 77, "147": 78, "149": 79, "150": 80, "151": 81, "152": 82, "153": 83, "154": 84, "157": 85, "159": 86, "161": 87, "163": 88, "166": 89, "167": 90, "168": 91, "173": 92, "174": 93, "175": 94, "176": 95, "177": 96, "188": 97, "192": 98, "196": 99, "198": 100, "199": 101, "200": 102, "201": 103, "202": 104, "203": 105, "204": 106, "205": 107, "207": 108, "208": 109, "209": 110, "210": 111, "211": 112, "215": 113, "216": 114, "217": 115, "218": 116, "224": 117, "225": 118, "226": 119, "227": 120, "228": 121, "229": 122, "231": 123, "233": 124, "237": 125, "238": 126, "240": 127, "242": 128, "243": 129, "245": 130, "246": 131, "248": 132, "249": 133, "250": 134, "251": 135, "252": 136, "253": 137, "254": 138, "255": 139, "258": 140, "260": 141, "263": 142, "264": 143, "265": 144, "272": 145, "273": 146, "274": 147, "277": 148, "278": 149, "283": 150, "288": 151, "289": 152, "294": 153, "296": 154, "298": 155, "302": 156, "303": 157, "307": 158, "310": 159, "311": 160, "313": 161, "317": 162, "318": 163, "319": 164, "322": 165, "323": 166, "328": 167, "329": 168, "331": 169, "332": 170, "335": 171, "336": 172, "337": 173, "339": 174, "340": 175, "345": 176, "348": 177, "353": 178, "359": 179, "362": 180, "365": 181, "366": 182, "367": 183, "369": 184, "373": 185, "374": 186, "377": 187, "380": 188, "392": 189, "398": 190, "402": 191, "403": 192, "404": 193, "405": 194, "408": 195, "409": 196, "412": 197, "413": 198, "421": 199, "422": 200, "426": 201, "428": 202, "432": 203, "434": 204, "439": 205, "441": 206, "444": 207, "445": 208, "446": 209, "448": 210, "451": 211, "453": 212, "454": 213, "458": 214, "459": 215, "460": 216, "464": 217, "466": 218, "470": 219, "472": 220, "474": 221, "475": 222, "476": 223, "479": 224, "480": 225, "481": 226, "483": 227, "487": 228, "489": 229, "492": 230, "497": 231, "500": 232, "501": 233, "505": 234, "510": 235, "511": 236, "512": 237, "517": 238, "525": 239, "533": 240, "534": 241, "542": 242, "543": 243, "544": 244, "548": 245, "549": 246, "551": 247, "557": 248, "559": 249, "561": 250, "567": 251, "568": 252, "569": 253, "572": 254, "576": 255, "580": 256, "581": 257, "583": 258, "584": 259, "585": 260, "587": 261, "589": 262, "593": 263, "594": 264, "596": 265, "597": 266, "598": 267, "606": 268, "608": 269, "612": 270, "613": 271, "614": 272, "622": 273, "625": 274, "636": 275, "637": 276, "639": 277, "644": 278, "652": 279, "663": 280, "664": 281, "666": 282, "667": 283, "669": 284, "671": 285, "672": 286, "679": 287, "681": 288, "684": 289, "688": 290, "690": 291, "696": 292, "698": 293, "699": 294, "700": 295, "705": 296, "707": 297, "708": 298, "711": 299, "712": 300, "713": 301, "716": 302, "718": 303, "720": 304, "724": 305, "726": 306, "727": 307, "728": 308, "730": 309, "731": 310, "737": 311, "742": 312, "753": 313, "764": 314, "766": 315, "770": 316, "774": 317, "777": 318, "778": 319, "779": 320, "780": 321, "781": 322, "782": 323, "783": 324, "789": 325, "791": 326, "792": 327, "797": 328, "803": 329, "806": 330, "807": 331, "810": 332, "811": 333, "815": 334, "816": 335, "820": 336, "826": 337, "829": 338, "830": 339, "831": 340, "834": 341, "835": 342, "836": 343, "839": 344, "844": 345, "845": 346, "846": 347, "850": 348, "851": 349, "868": 350, "876": 351, "882": 352, "884": 353, "886": 354, "887": 355, "895": 356, "899": 357, "908": 358, "909": 359, "911": 360, "915": 361, "920": 362, "921": 363, "922": 364, "923": 365, "925": 366, "927": 367, "937": 368, "948": 369, "949": 370, "951": 371, "953": 372, "954": 373, "956": 374, "957": 375, "960": 376, "964": 377, "968": 378, "969": 379, "976": 380, "978": 381, "979": 382, "982": 383, "984": 384, "985": 385, "986": 386, "1001": 387, "1006": 388, "1012": 389, "1018": 390, "1025": 391, "1027": 392, "1028": 393, "1031": 394, "1034": 395, "1040": 396, "1046": 397, "1049": 398, "1050": 399, "1051": 400, "1052": 401, "1053": 402, "1054": 403, "1058": 404, "1060": 405, "1061": 406, "1065": 407, "1066": 408, "1069": 409, "1079": 410, "1081": 411, "1084": 412, "1085": 413, "1088": 414, "1089": 415, "1092": 416, "1093": 417, "1094": 418, "1096": 419, "1097": 420, "1098": 421, "1100": 422, "1107": 423, "1110": 424, "1112": 425, "1116": 426, "1121": 427, "1124": 428, "1132": 429, "1152": 430, "1154": 431, "1160": 432, "1161": 433, "1165": 434, "1166": 435, "1168": 436, "1171": 437, "1175": 438, "1179": 439, "1182": 440, "1183": 441, "1184": 442, "1187": 443, "1188": 444, "1195": 445, "1200": 446, "1212": 447, "1221": 448, "1222": 449, "1224": 450, "1225": 451, "1226": 452, "1230": 453, "1235": 454, "1239": 455, "1241": 456, "1246": 457, "1250": 458, "1252": 459, "1255": 460, "1258": 461, "1259": 462, "1260": 463, "1261": 464, "1263": 465, "1264": 466, "1265": 467, "1266": 468, "1271": 469, "1272": 470, "1274": 471, "1280": 472, "1283": 473, "1284": 474, "1289": 475, "1290": 476, "1291": 477, "1296": 478, "1298": 479, "1311": 480, "1313": 481, "1316": 482, "1320": 483, "1322": 484, "1323": 485, "1331": 486, "1334": 487, "1335": 488, "1336": 489, "1337": 490, "1341": 491, "1342": 492, "1343": 493, "1347": 494, "1348": 495, "1349": 496, "1353": 497, "1355": 498, "1363": 499, "1365": 500, "1367": 501, "1370": 502, "1373": 503, "1374": 504, "1379": 505, "1382": 506, "1383": 507, "1384": 508, "1387": 509, "1390": 510, "1392": 511, "1401": 512, "1403": 513, "1413": 514, "1414": 515, "1417": 516, "1421": 517, "1422": 518, "1425": 519, "1430": 520, "1444": 521, "1445": 522, "1446": 523, "1447": 524, "1448": 525, "1455": 526, "1456": 527, "1460": 528, "1462": 529, "1463": 530, "1469": 531, "1472": 532, "1473": 533, "1474": 534, "1482": 535, "1485": 536, "1487": 537, "1492": 538, "1494": 539, "1495": 540, "1498": 541, "1502": 542, "1505": 543, "1509": 544, "1513": 545, "1535": 546, "1536": 547, "1544": 548, "1545": 549, "1547": 550, "1552": 551, "1553": 552, "1556": 553, "1559": 554, "1563": 555, "1564": 556, "1566": 557, "1569": 558, "1571": 559, "1572": 560, "1578": 561, "1579": 562, "1580": 563, "1585": 564, "1593": 565, "1594": 566, "1595": 567, "1601": 568, "1603": 569, "1607": 570, "1614": 571, "1618": 572, "1621": 573, "1624": 574, "1629": 575, "1630": 576, "1633": 577, "1634": 578, "1636": 579, "1638": 580, "1639": 581, "1641": 582, "1643": 583, "1645": 584, "1646": 585, "1647": 586, "1648": 587, "1649": 588, "1650": 589, "1651": 590, "1653": 591, "1664": 592, "1665": 593, "1668": 594, "1673": 595, "1674": 596, "1678": 597, "1679": 598, "1680": 599, "1681": 600, "1685": 601, "1686": 602, "1688": 603, "1690": 604, "1691": 605, "1693": 606, "1695": 607, "1696": 608, "1699": 609, "1701": 610, "1704": 611, "1705": 612, "1708": 613, "1710": 614, "1714": 615, "1715": 616, "1717": 617, "1721": 618, "1723": 619, "1724": 620, "1726": 621, "1731": 622, "1733": 623, "1734": 624, "1736": 625, "1737": 626, "1740": 627, "1743": 628, "1746": 629, "1748": 630, "1750": 631, "1752": 632, "1754": 633, "1756": 634, "1757": 635, "1760": 636, "1765": 637, "1767": 638, "1769": 639, "1772": 640, "1773": 641, "1776": 642, "1777": 643, "1779": 644, "1780": 645, "1784": 646, "1789": 647, "1795": 648, "1800": 649, "1801": 650, "1804": 651, "1806": 652, "1809": 653, "1811": 654, "1813": 655, "1815": 656, "1819": 657, "1825": 658, "1826": 659, "1827": 660, "1828": 661, "1841": 662, "1844": 663, "1845": 664, "1846": 665, "1849": 666, "1851": 667, "1859": 668, "1863": 669, "1867": 670, "1868": 671, "1870": 672, "1874": 673, "1878": 674, "1885": 675, "1898": 676, "1901": 677, "1903": 678, "1913": 679, "1914": 680, "1919": 681, "1920": 682, "1923": 683, "1924": 684, "1926": 685, "1931": 686, "1933": 687, "1938": 688, "1943": 689, "1944": 690, "1958": 691, "1961": 692, "1963": 693, "1968": 694, "1970": 695, "1974": 696, "1977": 697, "1985": 698, "1987": 699, "1988": 700, "1989": 701, "1992": 702, "1993": 703, "1995": 704, "1998": 705, "2001": 706, "2002": 707, "2003": 708, "2004": 709, "2007": 710, "2010": 711, "2012": 712, "2013": 713, "2021": 714, "2026": 715, "2033": 716, "2035": 717, "2039": 718, "2042": 719, "2045": 720, "2046": 721, "2050": 722, "2051": 723, "2053": 724, "2056": 725, "2060": 726, "2061": 727, "2062": 728, "2063": 729, "2067": 730, "2068": 731, "2074": 732, "2078": 733, "2085": 734, "2086": 735, "2089": 736, "2090": 737, "2092": 738, "2093": 739, "2094": 740, "2096": 741, "2100": 742, "2104": 743, "2110": 744, "2113": 745, "2122": 746, "2127": 747, "2133": 748, "2136": 749, "2137": 750, "2140": 751, "2143": 752, "2146": 753, "2148": 754, "2149": 755, "2152": 756, "2156": 757, "2159": 758, "2162": 759, "2167": 760, "2182": 761, "2185": 762, "2194": 763, "2195": 764, "2196": 765, "2198": 766, "2201": 767, "2204": 768, "2208": 769, "2229": 770, "2230": 771, "2234": 772, "2237": 773, "2238": 774, "2240": 775, "2246": 776, "2254": 777, "2256": 778, "2262": 779, "2269": 780, "2270": 781, "2272": 782, "2273": 783, "2275": 784, "2276": 785, "2277": 786, "2279": 787, "2284": 788, "2285": 789, "2288": 790, "2289": 791, "2292": 792, "2294": 793, "2297": 794, "2299": 795, "2300": 796, "2301": 797, "2309": 798, "2312": 799, "2319": 800, "2334": 801, "2339": 802, "2341": 803, "2346": 804, "2348": 805, "2351": 806, "2356": 807, "2361": 808, "2364": 809, "2368": 810, "2374": 811, "2380": 812, "2384": 813, "2388": 814, "2391": 815, "2393": 816, "2397": 817, "2401": 818, "2404": 819, "2405": 820, "2407": 821, "2411": 822, "2412": 823, "2414": 824, "2416": 825, "2427": 826, "2428": 827, "2436": 828, "2437": 829, "2445": 830, "2448": 831, "2473": 832, "2481": 833, "2485": 834, "2487": 835, "2488": 836, "2491": 837, "2494": 838, "2496": 839, "2498": 840, "2499": 841, "2504": 842, "2506": 843, "2512": 844, "2514": 845, "2517": 846, "2518": 847, "2522": 848, "2526": 849, "2531": 850, "2532": 851, "2533": 852, "2541": 853, "2544": 854, "2545": 855, "2552": 856, "2553": 857, "2562": 858, "2568": 859, "2570": 860, "2573": 861, "2574": 862, "2577": 863, "2581": 864, "2582": 865, "2587": 866, "2588": 867, "2589": 868, "2592": 869, "2598": 870, "2606": 871, "2607": 872, "2609": 873, "2618": 874, "2624": 875, "2625": 876, "2628": 877, "2638": 878, "2652": 879, "2654": 880, "2660": 881, "2671": 882, "2673": 883, "2674": 884, "2676": 885, "2688": 886, "2691": 887, "2694": 888, "2696": 889, "2709": 890, "2712": 891, "2724": 892, "2730": 893, "2733": 894, "2735": 895, "2740": 896, "2741": 897, "2748": 898, "2751": 899, "2754": 900, "2758": 901, "2762": 902, "2764": 903, "2769": 904, "2774": 905, "2775": 906, "2785": 907, "2787": 908, "2790": 909, "2792": 910, "2803": 911, "2812": 912, "2815": 913, "2816": 914, "2817": 915, "2823": 916, "2825": 917, "2827": 918, "2830": 919, "2834": 920, "2836": 921, "2843": 922, "2853": 923, "2854": 924, "2882": 925, "2893": 926, "2895": 927, "2902": 928, "2909": 929, "2910": 930, "2911": 931, "2919": 932, "2920": 933, "2925": 934, "2929": 935, "2930": 936, "2943": 937, "2946": 938, "2952": 939, "2960": 940, "2961": 941, "2967": 942, "2971": 943, "2975": 944, "2979": 945, "2985": 946, "2988": 947, "2989": 948, "2990": 949, "2992": 950, "2997": 951, "2998": 952, "2999": 953, "3000": 954, "3001": 955, "3003": 956, "3005": 957, "3006": 958, "3008": 959, "3009": 960, "3020": 961, "3021": 962, "3025": 963, "3032": 964, "3033": 965, "3045": 966, "3046": 967, "3053": 968, "3054": 969, "3060": 970, "3063": 971, "3070": 972, "3072": 973, "3079": 974, "3080": 975, "3081": 976, "3082": 977, "3083": 978, "3088": 979, "3090": 980, "3092": 981, "3094": 982, "3097": 983, "3098": 984, "3100": 985, "3105": 986, "3109": 987, "3112": 988, "3114": 989, "3118": 990, "3119": 991, "3125": 992, "3132": 993, "3135": 994, "3137": 995, "3138": 996, "3142": 997, "3143": 998, "3144": 999, "3148": 1000, "3157": 1001, "3168": 1002, "3170": 1003, "3171": 1004, "3172": 1005, "3179": 1006, "3180": 1007, "3185": 1008, "3187": 1009, "3192": 1010, "3196": 1011, "3214": 1012, "3215": 1013, "3221": 1014, "3224": 1015, "3227": 1016, "3228": 1017, "3230": 1018, "3235": 1019, "3238": 1020, "3240": 1021, "3242": 1022, "3244": 1023, "3245": 1024, "3257": 1025, "3258": 1026, "3259": 1027, "3261": 1028, "3268": 1029, "3271": 1030, "3272": 1031, "3274": 1032, "3285": 1033, "3288": 1034, "3289": 1035, "3290": 1036, "3294": 1037, "3307": 1038, "3314": 1039, "3318": 1040, "3319": 1041, "3328": 1042, "3330": 1043, "3331": 1044, "3334": 1045, "3340": 1046, "3346": 1047, "3347": 1048, "3356": 1049, "3357": 1050, "3361": 1051, "3368": 1052, "3370": 1053, "3373": 1054, "3374": 1055, "3379": 1056, "3380": 1057, "3381": 1058, "3389": 1059, "3394": 1060, "3400": 1061, "3409": 1062, "3411": 1063, "3417": 1064, "3433": 1065, "3436": 1066, "3440": 1067, "3446": 1068, "3448": 1069, "3465": 1070, "3467": 1071, "3470": 1072, "3479": 1073, "3482": 1074, "3483": 1075, "3486": 1076, "3488": 1077, "3490": 1078, "3493": 1079, "3500": 1080, "3503": 1081, "3513": 1082, "3521": 1083, "3526": 1084, "3528": 1085, "3536": 1086, "3537": 1087, "3538": 1088, "3540": 1089, "3541": 1090, "3546": 1091, "3547": 1092, "3549": 1093, "3551": 1094, "3553": 1095, "3554": 1096, "3557": 1097, "3559": 1098, "3564": 1099, "3567": 1100, "3570": 1101, "3571": 1102, "3575": 1103, "3576": 1104, "3584": 1105, "3587": 1106, "3588": 1107, "3592": 1108, "3595": 1109, "3598": 1110, "3606": 1111, "3607": 1112, "3615": 1113, "3618": 1114, "3630": 1115, "3638": 1116, "3641": 1117, "3645": 1118, "3647": 1119, "3650": 1120, "3654": 1121, "3656": 1122, "3657": 1123, "3660": 1124, "3663": 1125, "3664": 1126, "3665": 1127, "3675": 1128, "3679": 1129, "3681": 1130, "3686": 1131, "3691": 1132, "3698": 1133, "3699": 1134, "3703": 1135, "3717": 1136, "3723": 1137, "3728": 1138, "3729": 1139, "3733": 1140, "3738": 1141, "3744": 1142, "3747": 1143, "3752": 1144, "3757": 1145, "3764": 1146, "3779": 1147, "3780": 1148, "3781": 1149, "3783": 1150, "3790": 1151, "3792": 1152, "3793": 1153, "3796": 1154, "3798": 1155, "3807": 1156, "3816": 1157, "3819": 1158, "3825": 1159, "3830": 1160, "3835": 1161, "3843": 1162, "3845": 1163, "3848": 1164, "3851": 1165, "3852": 1166, "3853": 1167, "3857": 1168, "3864": 1169, "3866": 1170, "3867": 1171, "3869": 1172, "3871": 1173, "3876": 1174, "3879": 1175, "3885": 1176, "3889": 1177, "3894": 1178, "3895": 1179, "3896": 1180, "3905": 1181, "3906": 1182, "3909": 1183, "3911": 1184, "3912": 1185, "3914": 1186, "3915": 1187, "3922": 1188, "3923": 1189, "3925": 1190, "3926": 1191, "3927": 1192, "3928": 1193, "3934": 1194, "3945": 1195, "3947": 1196, "3955": 1197, "3959": 1198, "3962": 1199, "3967": 1200, "3969": 1201, "3972": 1202, "3977": 1203, "3979": 1204, "3982": 1205, "3983": 1206, "3989": 1207, "3990": 1208, "3992": 1209, "3994": 1210, "3997": 1211, "4005": 1212, "4009": 1213, "4010": 1214, "4013": 1215, "4014": 1216, "4015": 1217, "4017": 1218, "4018": 1219, "4019": 1220, "4020": 1221, "4021": 1222, "4034": 1223, "4039": 1224, "4042": 1225, "4044": 1226, "4051": 1227, "4054": 1228, "4057": 1229, "4059": 1230, "4063": 1231, "4064": 1232, "4071": 1233, "4077": 1234, "4078": 1235, "4085": 1236, "4088": 1237, "4090": 1238, "4098": 1239, "4104": 1240, "4108": 1241, "4110": 1242, "4111": 1243, "4116": 1244, "4122": 1245, "4133": 1246, "4137": 1247, "4138": 1248, "4145": 1249, "4148": 1250, "4152": 1251, "4153": 1252, "4156": 1253, "4160": 1254, "4161": 1255, "4172": 1256, "4174": 1257, "4179": 1258, "4189": 1259, "4191": 1260, "4192": 1261, "4193": 1262, "4195": 1263, "4196": 1264, "4198": 1265, "4205": 1266, "4211": 1267, "4214": 1268, "4216": 1269, "4217": 1270, "4218": 1271, "4222": 1272, "4225": 1273, "4226": 1274, "4234": 1275, "4235": 1276, "4236": 1277, "4238": 1278, "4243": 1279, "4246": 1280, "4257": 1281, "4260": 1282, "4262": 1283, "4263": 1284, "4267": 1285, "4273": 1286, "4277": 1287, "4278": 1288, "4280": 1289, "4289": 1290, "4290": 1291, "4294": 1292, "4295": 1293, "4297": 1294, "4305": 1295, "4310": 1296, "4313": 1297, "4321": 1298, "4323": 1299, "4327": 1300, "4331": 1301, "4335": 1302, "4340": 1303, "4344": 1304, "4345": 1305, "4350": 1306, "4352": 1307, "4356": 1308, "4358": 1309, "4362": 1310, "4363": 1311, "4379": 1312, "4381": 1313, "4396": 1314, "4397": 1315, "4402": 1316, "4406": 1317, "4407": 1318, "4411": 1319, "4415": 1320, "4420": 1321, "4422": 1322, "4423": 1323, "4425": 1324, "4427": 1325, "4428": 1326, "4433": 1327, "4434": 1328, "4438": 1329, "4441": 1330, "4442": 1331, "4443": 1332, "4446": 1333, "4447": 1334, "4455": 1335, "4463": 1336, "4474": 1337, "4480": 1338, "4481": 1339, "4484": 1340, "4487": 1341, "4490": 1342, "4492": 1343, "4495": 1344, "4507": 1345, "4511": 1346, "4513": 1347, "4515": 1348, "4519": 1349, "4520": 1350, "4535": 1351, "4545": 1352, "4546": 1353, "4549": 1354, "4563": 1355, "4570": 1356, "4572": 1357, "4576": 1358, "4583": 1359, "4586": 1360, "4590": 1361, "4591": 1362, "4592": 1363, "4594": 1364, "4595": 1365, "4598": 1366, "4599": 1367, "4629": 1368, "4640": 1369, "4652": 1370, "4659": 1371, "4660": 1372, "4667": 1373, "4680": 1374, "4681": 1375, "4687": 1376, "4693": 1377, "4697": 1378, "4699": 1379, "4701": 1380, "4703": 1381, "4705": 1382, "4706": 1383, "4710": 1384, "4712": 1385, "4719": 1386, "4731": 1387, "4733": 1388, "4734": 1389, "4738": 1390, "4741": 1391, "4742": 1392, "4744": 1393, "4748": 1394, "4750": 1395, "4757": 1396, "4766": 1397, "4767": 1398, "4770": 1399, "4771": 1400, "4773": 1401, "4779": 1402, "4788": 1403, "4791": 1404, "4799": 1405, "4800": 1406, "4806": 1407, "4807": 1408, "4813": 1409, "4821": 1410, "4824": 1411, "4830": 1412, "4831": 1413, "4836": 1414, "4837": 1415, "4839": 1416, "4841": 1417, "4846": 1418, "4848": 1419, "4852": 1420, "4853": 1421, "4854": 1422, "4856": 1423, "4859": 1424, "4860": 1425, "4863": 1426, "4872": 1427, "4894": 1428, "4898": 1429, "4899": 1430, "4910": 1431, "4915": 1432, "4926": 1433, "4930": 1434, "4931": 1435, "4936": 1436, "4945": 1437, "4948": 1438, "4955": 1439, "4957": 1440, "4958": 1441, "4959": 1442, "4964": 1443, "4965": 1444, "4967": 1445, "4969": 1446, "4970": 1447, "4973": 1448, "4979": 1449, "4991": 1450, "4992": 1451, "4993": 1452, "5000": 1453, "5002": 1454, "5005": 1455, "5007": 1456, "5009": 1457, "5012": 1458, "5013": 1459, "5019": 1460, "5022": 1461, "5023": 1462, "5029": 1463, "5036": 1464, "5038": 1465, "5039": 1466, "5043": 1467, "5044": 1468, "5045": 1469, "5049": 1470, "5054": 1471, "5060": 1472, "5062": 1473, "5063": 1474, "5076": 1475, "5077": 1476, "5082": 1477, "5092": 1478, "5093": 1479, "5101": 1480, "5104": 1481, "5105": 1482, "5115": 1483, "5118": 1484, "5123": 1485, "5126": 1486, "5132": 1487, "5133": 1488, "5136": 1489, "5139": 1490, "5141": 1491, "5142": 1492, "5147": 1493, "5152": 1494, "5154": 1495, "5157": 1496, "5163": 1497, "5164": 1498, "5172": 1499, "5181": 1500, "5183": 1501, "5185": 1502, "5186": 1503, "5189": 1504, "5190": 1505, "5192": 1506, "5198": 1507, "5199": 1508, "5206": 1509, "5217": 1510, "5220": 1511, "5224": 1512, "5230": 1513, "5233": 1514, "5239": 1515, "5242": 1516, "5244": 1517, "5245": 1518, "5246": 1519, "5248": 1520, "5252": 1521, "5261": 1522, "5266": 1523, "5269": 1524, "5271": 1525, "5278": 1526, "5280": 1527, "5285": 1528, "5287": 1529, "5290": 1530, "5293": 1531, "5296": 1532, "5299": 1533, "5304": 1534, "5319": 1535, "5321": 1536, "5322": 1537, "5325": 1538, "5328": 1539, "5333": 1540, "5337": 1541, "5338": 1542, "5339": 1543, "5340": 1544, "5350": 1545, "5355": 1546, "5361": 1547, "5375": 1548, "5379": 1549, "5386": 1550, "5389": 1551, "5390": 1552, "5393": 1553, "5400": 1554, "5401": 1555, "5405": 1556, "5412": 1557, "5424": 1558, "5429": 1559, "5439": 1560, "5442": 1561, "5445": 1562, "5448": 1563, "5456": 1564, "5459": 1565, "5460": 1566, "5463": 1567, "5468": 1568, "5471": 1569, "5480": 1570, "5484": 1571, "5487": 1572, "5489": 1573, "5506": 1574, "5513": 1575, "5514": 1576, "5519": 1577, "5536": 1578, "5538": 1579, "5543": 1580, "5545": 1581, "5561": 1582, "5565": 1583, "5567": 1584, "5569": 1585, "5570": 1586, "5583": 1587, "5588": 1588, "5604": 1589, "5606": 1590, "5618": 1591, "5620": 1592, "5622": 1593, "5628": 1594, "5635": 1595, "5636": 1596, "5637": 1597, "5639": 1598, "5641": 1599, "5649": 1600, "5652": 1601, "5653": 1602, "5655": 1603, "5656": 1604, "5660": 1605, "5661": 1606, "5665": 1607, "5671": 1608, "5672": 1609, "5678": 1610, "5682": 1611, "5683": 1612, "5684": 1613, "5688": 1614, "5694": 1615, "5700": 1616, "5703": 1617, "5712": 1618, "5717": 1619, "5719": 1620, "5720": 1621, "5723": 1622, "5724": 1623, "5725": 1624, "5727": 1625, "5731": 1626, "5733": 1627, "5735": 1628, "5740": 1629, "5746": 1630, "5750": 1631, "5756": 1632, "5764": 1633, "5765": 1634, "5767": 1635, "5772": 1636, "5776": 1637, "5778": 1638, "5781": 1639, "5784": 1640, "5789": 1641, "5791": 1642, "5796": 1643, "5802": 1644, "5808": 1645, "5809": 1646, "5810": 1647, "5825": 1648, "5826": 1649, "5831": 1650, "5837": 1651, "5840": 1652, "5849": 1653, "5854": 1654, "5860": 1655, "5867": 1656, "5868": 1657, "5874": 1658, "5876": 1659, "5883": 1660, "5886": 1661, "5890": 1662, "5893": 1663, "5894": 1664, "5895": 1665, "5906": 1666, "5909": 1667, "5910": 1668, "5911": 1669, "5913": 1670, "5914": 1671, "5918": 1672, "5929": 1673, "5933": 1674, "5935": 1675, "5940": 1676, "5949": 1677, "5951": 1678, "5952": 1679, "5968": 1680, "5970": 1681, "5975": 1682, "5977": 1683, "5979": 1684, "5980": 1685, "5983": 1686, "5984": 1687, "5985": 1688, "5993": 1689, "6000": 1690, "6003": 1691, "6006": 1692, "6009": 1693, "6010": 1694, "6014": 1695, "6019": 1696, "6025": 1697, "6030": 1698, "6032": 1699, "6035": 1700, "6037": 1701, "6038": 1702, "6051": 1703, "6054": 1704, "6060": 1705, "6064": 1706, "6065": 1707, "6070": 1708, "6072": 1709, "6075": 1710, "6076": 1711, "6077": 1712, "6078": 1713, "6080": 1714, "6081": 1715, "6082": 1716, "6084": 1717, "6087": 1718, "6088": 1719, "6097": 1720, "6098": 1721, "6099": 1722, "6102": 1723, "6104": 1724, "6106": 1725, "6111": 1726, "6115": 1727, "6119": 1728, "6120": 1729, "6121": 1730, "6123": 1731, "6126": 1732, "6127": 1733, "6128": 1734, "6131": 1735, "6135": 1736, "6138": 1737, "6139": 1738, "6145": 1739, "6147": 1740, "6153": 1741, "6157": 1742, "6159": 1743, "6160": 1744, "6167": 1745, "6173": 1746, "6177": 1747, "6178": 1748, "6181": 1749, "6184": 1750, "6188": 1751, "6189": 1752, "6196": 1753, "6199": 1754, "6206": 1755, "6209": 1756, "6211": 1757, "6215": 1758, "6221": 1759, "6224": 1760, "6227": 1761, "6232": 1762, "6233": 1763, "6235": 1764, "6236": 1765, "6241": 1766, "6242": 1767, "6248": 1768, "6249": 1769, "6251": 1770, "6254": 1771, "6258": 1772, "6267": 1773, "6269": 1774, "6272": 1775, "6276": 1776, "6281": 1777, "6284": 1778, "6286": 1779, "6288": 1780, "6294": 1781, "6295": 1782, "6300": 1783, "6308": 1784, "6311": 1785, "6313": 1786, "6317": 1787, "6319": 1788, "6323": 1789, "6324": 1790, "6330": 1791, "6332": 1792, "6333": 1793, "6339": 1794, "6341": 1795, "6345": 1796, "6346": 1797, "6351": 1798, "6352": 1799, "6353": 1800, "6356": 1801, "6358": 1802, "6359": 1803, "6364": 1804, "6367": 1805, "6368": 1806, "6370": 1807, "6371": 1808, "6373": 1809, "6377": 1810, "6378": 1811, "6385": 1812, "6388": 1813, "6391": 1814, "6395": 1815, "6399": 1816, "6402": 1817, "6406": 1818, "6407": 1819, "6411": 1820, "6415": 1821, "6418": 1822, "6426": 1823, "6432": 1824, "6436": 1825, "6437": 1826, "6446": 1827, "6454": 1828, "6455": 1829, "6458": 1830, "6459": 1831, "6467": 1832, "6476": 1833, "6482": 1834, "6484": 1835, "6488": 1836, "6492": 1837, "6494": 1838, "6497": 1839, "6499": 1840, "6505": 1841, "6506": 1842, "6509": 1843, "6510": 1844, "6512": 1845, "6513": 1846, "6518": 1847, "6519": 1848, "6529": 1849, "6531": 1850, "6533": 1851, "6534": 1852, "6535": 1853, "6538": 1854, "6539": 1855, "6540": 1856, "6544": 1857, "6548": 1858, "6549": 1859, "6550": 1860, "6553": 1861, "6555": 1862, "6557": 1863, "6563": 1864, "6567": 1865, "6568": 1866, "6574": 1867, "6575": 1868, "6583": 1869, "6590": 1870, "6594": 1871, "6599": 1872, "6609": 1873, "6610": 1874, "6614": 1875, "6620": 1876, "6625": 1877, "6627": 1878, "6636": 1879, "6637": 1880, "6641": 1881, "6643": 1882, "6652": 1883, "6660": 1884, "6668": 1885, "6670": 1886, "6673": 1887, "6674": 1888, "6676": 1889, "6683": 1890, "6685": 1891, "6686": 1892, "6687": 1893, "6689": 1894, "6690": 1895, "6694": 1896, "6695": 1897, "6696": 1898, "6701": 1899, "6705": 1900, "6707": 1901, "6709": 1902, "6713": 1903, "6724": 1904, "6726": 1905, "6727": 1906, "6733": 1907, "6735": 1908, "6741": 1909, "6743": 1910, "6746": 1911, "6747": 1912, "6749": 1913, "6752": 1914, "6753": 1915, "6754": 1916, "6758": 1917, "6763": 1918, "6773": 1919, "6777": 1920, "6782": 1921, "6784": 1922, "6788": 1923, "6792": 1924, "6794": 1925, "6798": 1926, "6804": 1927, "6807": 1928, "6818": 1929, "6821": 1930, "6828": 1931, "6829": 1932, "6836": 1933, "6841": 1934, "6846": 1935, "6848": 1936, "6849": 1937, "6853": 1938, "6865": 1939, "6875": 1940, "6877": 1941, "6880": 1942, "6882": 1943, "6883": 1944, "6892": 1945, "6895": 1946, "6902": 1947, "6904": 1948, "6906": 1949, "6912": 1950, "6913": 1951, "6914": 1952, "6918": 1953, "6923": 1954, "6924": 1955, "6925": 1956, "6927": 1957, "6930": 1958, "6937": 1959, "6938": 1960, "6943": 1961, "6945": 1962, "6947": 1963, "6950": 1964, "6951": 1965, "6954": 1966, "6956": 1967, "6962": 1968, "6963": 1969, "6965": 1970, "6967": 1971, "6974": 1972, "6978": 1973, "6981": 1974, "6993": 1975, "7000": 1976, "7001": 1977, "7008": 1978, "7009": 1979, "7011": 1980, "7012": 1981, "7018": 1982, "7021": 1983, "7026": 1984, "7030": 1985, "7046": 1986, "7051": 1987, "7055": 1988, "7059": 1989, "7061": 1990, "7062": 1991, "7065": 1992, "7067": 1993, "7069": 1994, "7073": 1995, "7078": 1996, "7079": 1997, "7085": 1998, "7090": 1999, "7092": 2000, "7095": 2001, "7096": 2002, "7097": 2003, "7105": 2004, "7107": 2005, "7113": 2006, "7117": 2007, "7120": 2008, "7121": 2009, "7125": 2010, "7126": 2011, "7127": 2012, "7128": 2013, "7131": 2014, "7134": 2015, "7135": 2016, "7138": 2017, "7139": 2018, "7140": 2019, "7143": 2020, "7145": 2021, "7147": 2022, "7148": 2023, "7150": 2024, "7155": 2025, "7169": 2026, "7170": 2027, "7176": 2028, "7177": 2029, "7178": 2030, "7188": 2031, "7189": 2032, "7190": 2033, "7197": 2034, "7198": 2035, "7199": 2036, "7205": 2037, "7208": 2038, "7215": 2039, "7218": 2040, "7220": 2041, "7223": 2042, "7226": 2043, "7228": 2044, "7229": 2045, "7238": 2046, "7239": 2047, "7240": 2048, "7241": 2049, "7242": 2050, "7245": 2051, "7246": 2052, "7247": 2053, "7250": 2054, "7255": 2055, "7258": 2056, "7263": 2057, "7264": 2058, "7265": 2059, "7276": 2060, "7277": 2061, "7278": 2062, "7285": 2063, "7286": 2064, "7294": 2065, "7297": 2066, "7299": 2067, "7301": 2068, "7302": 2069, "7307": 2070, "7312": 2071, "7313": 2072, "7314": 2073, "7315": 2074, "7316": 2075, "7318": 2076, "7320": 2077, "7326": 2078, "7327": 2079, "7331": 2080, "7333": 2081, "7335": 2082, "7337": 2083, "7338": 2084, "7339": 2085, "7342": 2086, "7346": 2087, "7348": 2088, "7354": 2089, "7357": 2090, "7360": 2091, "7367": 2092, "7376": 2093, "7383": 2094, "7384": 2095, "7387": 2096, "7389": 2097, "7391": 2098, "7392": 2099, "7395": 2100, "7398": 2101, "7402": 2102, "7408": 2103, "7416": 2104, "7423": 2105, "7424": 2106, "7433": 2107, "7434": 2108, "7436": 2109, "7437": 2110, "7445": 2111, "7447": 2112, "7448": 2113, "7460": 2114, "7463": 2115, "7467": 2116, "7475": 2117, "7478": 2118, "7480": 2119, "7481": 2120, "7484": 2121, "7491": 2122, "7492": 2123, "7495": 2124, "7498": 2125, "7502": 2126, "7505": 2127, "7507": 2128, "7510": 2129, "7511": 2130, "7512": 2131, "7514": 2132, "7515": 2133, "7517": 2134, "7518": 2135, "7520": 2136, "7522": 2137, "7525": 2138, "7538": 2139, "7540": 2140, "7552": 2141, "7553": 2142, "7555": 2143, "7556": 2144, "7558": 2145, "7559": 2146, "7561": 2147, "7565": 2148, "7569": 2149, "7584": 2150, "7585": 2151, "7594": 2152, "7597": 2153, "7601": 2154, "7603": 2155, "7607": 2156, "7608": 2157, "7609": 2158, "7618": 2159, "7635": 2160, "7640": 2161, "7641": 2162, "7644": 2163, "7647": 2164, "7649": 2165, "7654": 2166, "7657": 2167, "7665": 2168, "7672": 2169, "7679": 2170, "7683": 2171, "7687": 2172, "7688": 2173, "7691": 2174, "7697": 2175, "7699": 2176, "7700": 2177, "7702": 2178, "7704": 2179, "7705": 2180, "7708": 2181, "7713": 2182, "7717": 2183, "7720": 2184, "7729": 2185, "7730": 2186, "7732": 2187, "7733": 2188, "7737": 2189, "7739": 2190, "7746": 2191, "7749": 2192, "7752": 2193, "7754": 2194, "7756": 2195, "7762": 2196, "7764": 2197, "7766": 2198, "7769": 2199, "7777": 2200, "7780": 2201, "7783": 2202, "7786": 2203, "7789": 2204, "7794": 2205, "7795": 2206, "7796": 2207, "7800": 2208, "7802": 2209, "7809": 2210, "7816": 2211, "7823": 2212, "7825": 2213, "7826": 2214, "7828": 2215, "7832": 2216, "7833": 2217, "7835": 2218, "7837": 2219, "7839": 2220, "7843": 2221, "7848": 2222, "7850": 2223, "7859": 2224, "7867": 2225, "7868": 2226, "7871": 2227, "7874": 2228, "7879": 2229, "7881": 2230, "7883": 2231, "7886": 2232, "7892": 2233, "7898": 2234, "7902": 2235, "7909": 2236, "7910": 2237, "7912": 2238, "7923": 2239, "7925": 2240, "7926": 2241, "7932": 2242, "7933": 2243, "7938": 2244, "7939": 2245, "7942": 2246, "7945": 2247, "7946": 2248, "7949": 2249, "7956": 2250, "7957": 2251, "7959": 2252, "7962": 2253, "7967": 2254, "7975": 2255, "7976": 2256, "7981": 2257, "7982": 2258, "7988": 2259, "7991": 2260, "7994": 2261, "7995": 2262, "7997": 2263, "8005": 2264, "8006": 2265, "8008": 2266, "8009": 2267, "8011": 2268, "8012": 2269, "8014": 2270, "8015": 2271, "8023": 2272, "8028": 2273, "8033": 2274, "8040": 2275, "8042": 2276, "8044": 2277, "8050": 2278, "8051": 2279, "8057": 2280, "8058": 2281, "8063": 2282, "8066": 2283, "8071": 2284, "8072": 2285, "8075": 2286, "8080": 2287, "8087": 2288, "8088": 2289, "8095": 2290, "8097": 2291, "8098": 2292, "8108": 2293, "8112": 2294, "8113": 2295, "8118": 2296, "8119": 2297, "8123": 2298, "8131": 2299, "8138": 2300, "8142": 2301, "8143": 2302, "8148": 2303, "8152": 2304, "8156": 2305, "8163": 2306, "8164": 2307, "8168": 2308, "8169": 2309, "8172": 2310, "8173": 2311, "8176": 2312, "8180": 2313, "8183": 2314, "8188": 2315, "8190": 2316, "8193": 2317, "8194": 2318, "8195": 2319, "8197": 2320, "8199": 2321, "8200": 2322, "8208": 2323, "8215": 2324, "8222": 2325, "8224": 2326, "8225": 2327, "8226": 2328, "8228": 2329, "8230": 2330, "8238": 2331, "8240": 2332, "8242": 2333, "8245": 2334, "8246": 2335, "8250": 2336, "8254": 2337, "8259": 2338, "8262": 2339, "8266": 2340, "8272": 2341, "8273": 2342, "8280": 2343, "8288": 2344, "8291": 2345, "8295": 2346, "8296": 2347, "8297": 2348, "8300": 2349, "8302": 2350, "8307": 2351, "8312": 2352, "8316": 2353, "8321": 2354, "8322": 2355, "8324": 2356, "8328": 2357, "8329": 2358, "8334": 2359, "8337": 2360, "8346": 2361, "8347": 2362, "8356": 2363, "8367": 2364, "8382": 2365, "8388": 2366, "8389": 2367, "8392": 2368, "8394": 2369, "8396": 2370, "8401": 2371, "8404": 2372, "8410": 2373, "8413": 2374, "8414": 2375, "8415": 2376, "8419": 2377, "8421": 2378, "8422": 2379, "8424": 2380, "8425": 2381, "8430": 2382, "8432": 2383, "8441": 2384, "8443": 2385, "8444": 2386, "8445": 2387, "8447": 2388, "8455": 2389, "8459": 2390, "8461": 2391, "8463": 2392, "8464": 2393, "8465": 2394, "8466": 2395, "8468": 2396, "8470": 2397, "8474": 2398, "8476": 2399, "8479": 2400, "8490": 2401, "8494": 2402, "8498": 2403, "8499": 2404, "8500": 2405, "8506": 2406, "8527": 2407, "8531": 2408, "8534": 2409, "8536": 2410, "8543": 2411, "8544": 2412, "8545": 2413, "8555": 2414, "8565": 2415, "8573": 2416, "8575": 2417, "8576": 2418, "8580": 2419, "8587": 2420, "8590": 2421, "8591": 2422, "8592": 2423, "8605": 2424, "8609": 2425, "8619": 2426, "8625": 2427, "8629": 2428, "8630": 2429, "8631": 2430, "8632": 2431, "8635": 2432, "8643": 2433, "8644": 2434, "8664": 2435, "8666": 2436, "8671": 2437, "8675": 2438, "8677": 2439, "8678": 2440, "8684": 2441, "8687": 2442, "8699": 2443, "8705": 2444, "8710": 2445, "8713": 2446, "8718": 2447, "8722": 2448, "8725": 2449, "8742": 2450, "8747": 2451, "8753": 2452, "8758": 2453, "8765": 2454, "8770": 2455, "8771": 2456, "8772": 2457, "8776": 2458, "8778": 2459, "8786": 2460, "8791": 2461, "8797": 2462, "8799": 2463, "8803": 2464, "8808": 2465, "8820": 2466, "8824": 2467, "8825": 2468, "8838": 2469, "8842": 2470, "8846": 2471, "8848": 2472, "8855": 2473, "8867": 2474, "8875": 2475, "8879": 2476, "8887": 2477, "8897": 2478, "8975": 2479, "9000": 2480, "9022": 2481, "9023": 2482, "9026": 2483} \ No newline at end of file diff --git a/config/speakers.json b/config/speakers.json new file mode 100644 index 0000000000000000000000000000000000000000..535dfbd4bcc33f89ab53244c912ce0fbfd6def54 --- /dev/null +++ b/config/speakers.json @@ -0,0 +1 @@ +{"3": {"id": 3, "name": "Kara Shallenberg", "gender": "F"}, "8": {"id": 8, "name": "Denny Sayers", "gender": "M"}, "9": {"id": 9, "name": "Sean McKinley", "gender": "M"}, "14": {"id": 14, "name": "Betsie Bush", "gender": "F"}, "18": {"id": 18, "name": "Sherry Crowther", "gender": "F"}, "19": {"id": 19, "name": "Vicki Barbour", "gender": "F"}, "32": {"id": 32, "name": "nan", "gender": "M"}, "41": {"id": 41, "name": "Hugh McGuire", "gender": "M"}, "45": {"id": 45, "name": "Catharine Eastman", "gender": "F"}, "48": {"id": 48, "name": "Rosalind Wills", "gender": "F"}, "49": {"id": 49, "name": "Kristen McQuillin", "gender": "F"}, "59": {"id": 59, "name": "Karen Savage", "gender": "F"}, "68": {"id": 68, "name": "Alex Buie", "gender": "M"}, "73": {"id": 73, "name": "Claire Goget", "gender": "F"}, "80": {"id": 80, "name": "Fox in the Stars", "gender": "F"}, "88": {"id": 88, "name": "Andrew Miller", "gender": "M"}, "99": {"id": 99, "name": "Stewart Wills", "gender": "M"}, "100": {"id": 100, "name": "Heather Barnett", "gender": "F"}, "102": {"id": 102, "name": "Maureen S. O'Brien", "gender": "F"}, "103": {"id": 103, "name": "Joplin James", "gender": "M"}, "112": {"id": 112, "name": "shanda_w", "gender": "F"}, "119": {"id": 119, "name": "Deb Bacon-Ziegler", "gender": "F"}, "122": {"id": 122, "name": "carnright", "gender": "M"}, "124": {"id": 124, "name": "Steve Karafit", "gender": "M"}, "132": {"id": 132, "name": "Becky Miller", "gender": "F"}, "134": {"id": 134, "name": "Mary Reagan", "gender": "F"}, "138": {"id": 138, "name": "Alan Davis Drake (1945-2010)", "gender": "M"}, "152": {"id": 152, "name": "Barbara Wedge", "gender": "F"}, "155": {"id": 155, "name": "Caroline Morse", "gender": "F"}, "156": {"id": 156, "name": "Chris Peterson", "gender": "F"}, "158": {"id": 158, "name": "Randy Phillips", "gender": "M"}, "160": {"id": 160, "name": "deadwhitemales", "gender": "M"}, "165": {"id": 165, "name": "Elisabeth Shields", "gender": "F"}, "167": {"id": 167, "name": "Elizabeth Palmer", "gender": "F"}, "170": {"id": 170, "name": "Aaron Teiser", "gender": "M"}, "186": {"id": 186, "name": "kumarei", "gender": "M"}, "192": {"id": 192, "name": "Nocturna", "gender": "F"}, "194": {"id": 194, "name": "Eric Dennison", "gender": "M"}, "197": {"id": 197, "name": "Brian Roberg", "gender": "M"}, "201": {"id": 201, "name": "Norah Piehl", "gender": "F"}, "206": {"id": 206, "name": "Sandra in Wales, United Kingdom", "gender": "F"}, "208": {"id": 208, "name": "Dave Foss", "gender": "M"}, "209": {"id": 209, "name": "Steve Hartzog", "gender": "M"}, "214": {"id": 214, "name": "Scott Splavec", "gender": "M"}, "216": {"id": 216, "name": "Dave Ranson", "gender": "M"}, "226": {"id": 226, "name": "Neal Foley", "gender": "M"}, "261": {"id": 261, "name": "Joy Scaglione", "gender": "F"}, "274": {"id": 274, "name": "toriasuncle", "gender": "M"}, "284": {"id": 284, "name": "Anne", "gender": "F"}, "292": {"id": 292, "name": "Tamara R. Schwartz", "gender": "F"}, "309": {"id": 309, "name": "Karen Labenz", "gender": "F"}, "340": {"id": 340, "name": "Nick Gallant", "gender": "M"}, "344": {"id": 344, "name": "rovert405", "gender": "M"}, "355": {"id": 355, "name": "Lana Taylor", "gender": "F"}, "359": {"id": 359, "name": "Greg Bryant", "gender": "M"}, "360": {"id": 360, "name": "frankjf", "gender": "M"}, "395": {"id": 395, "name": "Kevin O'Coin", "gender": "M"}, "396": {"id": 396, "name": "John Garvin", "gender": "M"}, "409": {"id": 409, "name": "Dawn", "gender": "F"}, "411": {"id": 411, "name": "Fracture", "gender": "M"}, "414": {"id": 414, "name": "Christabel", "gender": "F"}, "421": {"id": 421, "name": "Merryb", "gender": "F"}, "426": {"id": 426, "name": "Megan Stemm-Wade", "gender": "F"}, "441": {"id": 441, "name": "roolynninms", "gender": "F"}, "454": {"id": 454, "name": "Tim Gregory", "gender": "M"}, "457": {"id": 457, "name": "Sandra", "gender": "F"}, "465": {"id": 465, "name": "Leonie Rose", "gender": "F"}, "487": {"id": 487, "name": "John Schell", "gender": "M"}, "498": {"id": 498, "name": "Chris Gladis", "gender": "M"}, "499": {"id": 499, "name": "Tammy Sanders", "gender": "F"}, "524": {"id": 524, "name": "Luigina", "gender": "F"}, "526": {"id": 526, "name": "webslog", "gender": "M"}, "542": {"id": 542, "name": "Ann Boyer", "gender": "F"}, "552": {"id": 552, "name": "Mim Ritty", "gender": "F"}, "561": {"id": 561, "name": "Lorelle Anderson", "gender": "F"}, "566": {"id": 566, "name": "Jon Scott Jones", "gender": "M"}, "574": {"id": 574, "name": "Daniel Shorten", "gender": "M"}, "619": {"id": 619, "name": "Rob Whelan", "gender": "M"}, "626": {"id": 626, "name": "Erin Hastings", "gender": "F"}, "628": {"id": 628, "name": "Bryan Ness", "gender": "M"}, "662": {"id": 662, "name": "Laura Caldwell", "gender": "F"}, "670": {"id": 670, "name": "Rowdy Delaney", "gender": "M"}, "676": {"id": 676, "name": "Jennifer", "gender": "F"}, "685": {"id": 685, "name": "Nikki Sullivan", "gender": "F"}, "693": {"id": 693, "name": "Belinda Brown", "gender": "F"}, "695": {"id": 695, "name": "Dawn Larsen", "gender": "F"}, "702": {"id": 702, "name": "Michelle White", "gender": "F"}, "707": {"id": 707, "name": "Larry Maddocks", "gender": "M"}, "710": {"id": 710, "name": "Sheila Morton", "gender": "F"}, "738": {"id": 738, "name": "Elaine Hamby", "gender": "F"}, "749": {"id": 749, "name": "Great Plains", "gender": "M"}, "758": {"id": 758, "name": "Matthew Westra", "gender": "M"}, "761": {"id": 761, "name": "Susan Umpleby", "gender": "F"}, "765": {"id": 765, "name": "Andrea Fiore", "gender": "F"}, "791": {"id": 791, "name": "David Kleparek", "gender": "M"}, "813": {"id": 813, "name": "Ger", "gender": "M"}, "815": {"id": 815, "name": "treefingers", "gender": "F"}, "825": {"id": 825, "name": "Julia Albath", "gender": "F"}, "828": {"id": 828, "name": "Seth Adam Sher", "gender": "M"}, "845": {"id": 845, "name": "S. Young", "gender": "M"}, "847": {"id": 847, "name": "Rob Powell", "gender": "M"}, "887": {"id": 887, "name": "Donna Stewart", "gender": "F"}, "903": {"id": 903, "name": "Piper Hale", "gender": "F"}, "915": {"id": 915, "name": "Catherine Millward", "gender": "F"}, "921": {"id": 921, "name": "Linda McDaniel", "gender": "F"}, "922": {"id": 922, "name": "ricell", "gender": "M"}, "926": {"id": 926, "name": "Ryan Sutter", "gender": "M"}, "930": {"id": 930, "name": "Janna", "gender": "F"}, "931": {"id": 931, "name": "David Lawrence", "gender": "M"}, "939": {"id": 939, "name": "Scott Carpenter", "gender": "M"}, "948": {"id": 948, "name": "Jamie Strassenburg, Cypress, California", "gender": "F"}, "988": {"id": 988, "name": "Jessica Louise", "gender": "F"}, "1002": {"id": 1002, "name": "David Anton", "gender": "M"}, "1012": {"id": 1012, "name": "fourteatoo", "gender": "M"}, "1019": {"id": 1019, "name": "Karen Commins", "gender": "F"}, "1021": {"id": 1021, "name": "flakker", "gender": "M"}, "1022": {"id": 1022, "name": "peac", "gender": "M"}, "1027": {"id": 1027, "name": "Kate West", "gender": "F"}, "1055": {"id": 1055, "name": "Craig Campbell", "gender": "M"}, "1066": {"id": 1066, "name": "Anders Lankford", "gender": "M"}, "1067": {"id": 1067, "name": "Heidi Will", "gender": "F"}, "1076": {"id": 1076, "name": "Robin Balmer", "gender": "M"}, "1084": {"id": 1084, "name": "Bereni", "gender": "F"}, "1112": {"id": 1112, "name": "Richard Wallis", "gender": "M"}, "1126": {"id": 1126, "name": "Barry Eads", "gender": "M"}, "1134": {"id": 1134, "name": "Bruce Pirie", "gender": "M"}, "1137": {"id": 1137, "name": "Kevin Lavin", "gender": "M"}, "1156": {"id": 1156, "name": "Jesse Noar", "gender": "M"}, "1160": {"id": 1160, "name": "rymd80", "gender": "M"}, "1168": {"id": 1168, "name": "Epistomolus", "gender": "M"}, "1175": {"id": 1175, "name": "Keneva", "gender": "F"}, "1196": {"id": 1196, "name": "johnell", "gender": "F"}, "1205": {"id": 1205, "name": "Kate Adams", "gender": "F"}, "1206": {"id": 1206, "name": "lavocedorata", "gender": "F"}, "1216": {"id": 1216, "name": "Tom Clifton", "gender": "M"}, "1219": {"id": 1219, "name": "Nicholas Clifford", "gender": "M"}, "1227": {"id": 1227, "name": "Liz Devens", "gender": "F"}, "1237": {"id": 1237, "name": "Blazin48", "gender": "F"}, "1247": {"id": 1247, "name": "Sarah LuAnn", "gender": "F"}, "1254": {"id": 1254, "name": "Rosie", "gender": "F"}, "1263": {"id": 1263, "name": "bj", "gender": "F"}, "1268": {"id": 1268, "name": "A. Janelle Risa", "gender": "F"}, "1285": {"id": 1285, "name": "Ric F", "gender": "M"}, "1294": {"id": 1294, "name": "Tina Horning", "gender": "F"}, "1303": {"id": 1303, "name": "kiwafruit", "gender": "F"}, "1310": {"id": 1310, "name": "Michelle Montano", "gender": "F"}, "1315": {"id": 1315, "name": "John Dennison", "gender": "M"}, "1317": {"id": 1317, "name": "Matthew Scott Surprenant", "gender": "M"}, "1330": {"id": 1330, "name": "William Peck", "gender": "M"}, "1339": {"id": 1339, "name": "margo zinberg", "gender": "F"}, "1369": {"id": 1369, "name": "Karen Mason", "gender": "F"}, "1374": {"id": 1374, "name": "pachayes", "gender": "F"}, "1403": {"id": 1403, "name": "Bill Boerst", "gender": "M"}, "1409": {"id": 1409, "name": "Steve Mattern", "gender": "M"}, "1412": {"id": 1412, "name": "George Aalto", "gender": "M"}, "1421": {"id": 1421, "name": "Barbara Derksen", "gender": "F"}, "1424": {"id": 1424, "name": "nathank", "gender": "M"}, "1429": {"id": 1429, "name": "greatbasinrain", "gender": "M"}, "1461": {"id": 1461, "name": "Kathleen Costa", "gender": "F"}, "1470": {"id": 1470, "name": "Bradley Smith", "gender": "M"}, "1481": {"id": 1481, "name": "Chuck Burke", "gender": "M"}, "1497": {"id": 1497, "name": "LilyAnne", "gender": "F"}, "1506": {"id": 1506, "name": "Jason Esteves", "gender": "M"}, "1537": {"id": 1537, "name": "Jay Bidal", "gender": "M"}, "1543": {"id": 1543, "name": "Lauren McCullough", "gender": "F"}, "1552": {"id": 1552, "name": "Charles Bice", "gender": "M"}, "1553": {"id": 1553, "name": "Amy Hengst", "gender": "F"}, "1564": {"id": 1564, "name": "e_scarab", "gender": "M"}, "1567": {"id": 1567, "name": "GLM", "gender": "M"}, "1576": {"id": 1576, "name": "Ella Jane Quentin", "gender": "F"}, "1582": {"id": 1582, "name": "Ellen Jones", "gender": "F"}, "1601": {"id": 1601, "name": "amicrazy2u", "gender": "F"}, "1610": {"id": 1610, "name": "jgoffena", "gender": "M"}, "1614": {"id": 1614, "name": "Jennifer Dionne", "gender": "F"}, "1617": {"id": 1617, "name": "Garth Comira", "gender": "M"}, "1631": {"id": 1631, "name": "laurencetrask", "gender": "M"}, "1638": {"id": 1638, "name": "Laura Victoria", "gender": "F"}, "1641": {"id": 1641, "name": "Kirsten Wever", "gender": "F"}, "1645": {"id": 1645, "name": "jeandelfrio", "gender": "M"}, "1656": {"id": 1656, "name": "Sharon Omi", "gender": "F"}, "1690": {"id": 1690, "name": "MissRose", "gender": "F"}, "1696": {"id": 1696, "name": "DerekP", "gender": "M"}, "1706": {"id": 1706, "name": "Deborah Knight", "gender": "F"}, "1713": {"id": 1713, "name": "dobsonfly", "gender": "F"}, "1715": {"id": 1715, "name": "Lazuli", "gender": "M"}, "1740": {"id": 1740, "name": "Liberty Stump", "gender": "F"}, "1749": {"id": 1749, "name": "Mike", "gender": "M"}, "1756": {"id": 1756, "name": "deckerteach", "gender": "M"}, "1775": {"id": 1775, "name": "jlenardon", "gender": "F"}, "1805": {"id": 1805, "name": "Vince Dee", "gender": "M"}, "1812": {"id": 1812, "name": "Novella Serena", "gender": "F"}, "1821": {"id": 1821, "name": "Daryl Wor", "gender": "F"}, "1826": {"id": 1826, "name": "John Hoerr", "gender": "M"}, "1828": {"id": 1828, "name": "David Wales", "gender": "M"}, "1833": {"id": 1833, "name": "Viridian", "gender": "F"}, "1849": {"id": 1849, "name": "Fred DeBerardinis", "gender": "M"}, "1850": {"id": 1850, "name": "janesandberg", "gender": "F"}, "1864": {"id": 1864, "name": "William Tomcho", "gender": "M"}, "1929": {"id": 1929, "name": "beckyboyd", "gender": "F"}, "1933": {"id": 1933, "name": "John", "gender": "M"}, "1936": {"id": 1936, "name": "KarlHenning", "gender": "M"}, "1942": {"id": 1942, "name": "Capybara", "gender": "M"}, "1956": {"id": 1956, "name": "Thomas Meaney", "gender": "M"}, "1989": {"id": 1989, "name": "Joannemmp", "gender": "F"}, "1993": {"id": 1993, "name": "Matthew Wall", "gender": "M"}, "1996": {"id": 1996, "name": "Mary in Arkansas", "gender": "F"}, "2006": {"id": 2006, "name": "Sukaina Jaffer", "gender": "F"}, "2023": {"id": 2023, "name": "Vickie Ranz", "gender": "F"}, "2030": {"id": 2030, "name": "J.K. Neely", "gender": "F"}, "2033": {"id": 2033, "name": "Tony Posante", "gender": "M"}, "2043": {"id": 2043, "name": "Jonathan Moore", "gender": "M"}, "2058": {"id": 2058, "name": "Sean McClain", "gender": "M"}, "2062": {"id": 2062, "name": "Jon Smith", "gender": "M"}, "2069": {"id": 2069, "name": "Asta1234", "gender": "F"}, "2071": {"id": 2071, "name": "nkneer", "gender": "M"}, "2092": {"id": 2092, "name": "NIneFive83", "gender": "M"}, "2102": {"id": 2102, "name": "Canby Ibarra", "gender": "M"}, "2112": {"id": 2112, "name": "dasbury", "gender": "M"}, "2127": {"id": 2127, "name": "Ron Lockhart", "gender": "M"}, "2130": {"id": 2130, "name": "Sherri Vance", "gender": "F"}, "2134": {"id": 2134, "name": "Raz Mason", "gender": "F"}, "2160": {"id": 2160, "name": "Judy Guinan", "gender": "F"}, "2201": {"id": 2201, "name": "tazzle", "gender": "F"}, "2205": {"id": 2205, "name": "mlcui", "gender": "F"}, "2208": {"id": 2208, "name": "Arie", "gender": "F"}, "2224": {"id": 2224, "name": "xinamarieuhl", "gender": "F"}, "2270": {"id": 2270, "name": "constatine", "gender": "F"}, "2279": {"id": 2279, "name": "Maria Kasper", "gender": "F"}, "2282": {"id": 2282, "name": "Robert Snoza", "gender": "M"}, "2289": {"id": 2289, "name": "Jason Bolestridge", "gender": "M"}, "2290": {"id": 2290, "name": "Theodulf", "gender": "M"}, "2292": {"id": 2292, "name": "Arnold", "gender": "M"}, "2293": {"id": 2293, "name": "drakaunus", "gender": "M"}, "2298": {"id": 2298, "name": "Sheila Wood", "gender": "F"}, "2328": {"id": 2328, "name": "Adam Picot", "gender": "M"}, "2331": {"id": 2331, "name": "Madam Fickle", "gender": "F"}, "2352": {"id": 2352, "name": "Jaimie Noy", "gender": "F"}, "2356": {"id": 2356, "name": "Kathy Wright", "gender": "F"}, "2377": {"id": 2377, "name": "Jon Kissack", "gender": "M"}, "2381": {"id": 2381, "name": "Larry Wilson", "gender": "M"}, "2394": {"id": 2394, "name": "TinaNygard2", "gender": "F"}, "2396": {"id": 2396, "name": "Jennifer Dorr", "gender": "F"}, "2419": {"id": 2419, "name": "Gary Dana", "gender": "M"}, "2425": {"id": 2425, "name": "noblesavage", "gender": "M"}, "2428": {"id": 2428, "name": "Shivansh Dhar", "gender": "M"}, "2429": {"id": 2429, "name": "Eduardo", "gender": "M"}, "2451": {"id": 2451, "name": "DeanOBuchanan", "gender": "M"}, "2455": {"id": 2455, "name": "Paul Simonin", "gender": "M"}, "2462": {"id": 2462, "name": "Sean Grabosky", "gender": "M"}, "2469": {"id": 2469, "name": "Kevin Owens", "gender": "M"}, "2479": {"id": 2479, "name": "Daisy Flaim", "gender": "F"}} \ No newline at end of file diff --git a/config/speakers.tsv b/config/speakers.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c5e1dfdfc1a1f72a13cb779d5e1affb752a5210e --- /dev/null +++ b/config/speakers.tsv @@ -0,0 +1,2485 @@ +READER GENDER SUBSET NAME +14 F train-clean-360 Kristin LeMoine +16 F train-clean-360 Alys AtteWater +17 M train-clean-360 Gord Mackenzie +19 F train-clean-100 Kara Shallenberg +20 F train-other-500 Gesine +22 F train-clean-360 Michelle Crandall +23 F train-clean-360 Anita Roy Dobbs +25 M train-other-500 John Gonzalez +26 M train-clean-100 Denny Sayers +27 M train-clean-100 Sean McKinley +28 F train-clean-360 Kristin Hughes +29 M train-other-500 Linton +30 F train-clean-360 Annie Coleman Rothenberg +31 M train-other-500 Martin Clifton +32 F train-clean-100 Betsie Bush +36 M train-other-500 Chip +37 M train-other-500 wolvrin +38 M train-clean-360 R. Francis Smith +39 F train-clean-100 Sherry Crowther +40 F train-clean-100 Vicki Barbour +44 F train-other-500 travelbratd +45 F train-other-500 Kelly Bescherer +46 M train-other-500 Aaron Hochwimmer +47 F train-other-500 Amanda +49 M train-other-500 vrk74 +51 M train-other-500 rakkar +52 F train-other-500 Luisa Hall +54 F train-clean-360 Westwinds12 +55 M train-clean-360 David Jaquay +56 F train-clean-360 Kirsten Ferreri +57 F train-other-500 Ophelia Darcy +58 M train-other-500 George Coutts +60 M train-clean-100 +61 M test-clean Paul-Gabriel Wiener +62 M train-other-500 Faris +64 F train-clean-360 Robin Cotter +65 F train-other-500 chriss the girl +66 M train-other-500 Alex Foster +70 M train-clean-360 Kurt Copeland +75 M train-other-500 Jim Mowatt +77 F train-other-500 JemmaBlythe +78 M train-clean-100 Hugh McGuire +79 F train-clean-360 Jennifer Crispin +81 M train-clean-360 Aaron Decker +82 F train-other-500 Marlo Dianne +83 F train-clean-100 Catharine Eastman +84 F dev-clean Christie Nowak +85 M train-other-500 David Leaman +87 F train-clean-100 Rosalind Wills +89 F train-clean-100 Kristen McQuillin +90 M train-clean-360 Dan Threetrees +91 M train-other-500 Stephan Mobius +92 F train-other-500 Cori Samuel +93 F train-clean-360 Kathy +94 M train-other-500 David Barnes +98 F train-clean-360 Patricia Oakley +100 F train-clean-360 Judy Bieber +101 M train-clean-360 paulino +102 F train-other-500 Linda Leu +103 F train-clean-100 Karen Savage +104 F train-other-500 Laura M.D. +107 M train-other-500 John Greenman +110 F train-other-500 Cynthia Lyons (1946-2011) +111 F train-other-500 Linda Wilcox +112 F train-clean-360 Christiane Levesque +114 F train-clean-360 Jen Kidd +115 F train-clean-360 Maddie +116 M dev-other Steven Collins +118 M train-clean-100 Alex Buie +119 M train-clean-360 Alex Patterson +121 F test-clean Nikolle Doolin +122 M train-clean-360 J.C. +123 F train-other-500 Ezwa +125 F train-clean-100 Claire Goget +126 F train-clean-360 Susan Denney +127 M train-other-500 John Hicken +128 M train-other-500 ML Cohen +133 M train-other-500 Mick +147 M train-other-500 Thomas Hoover +149 M train-other-500 Joshua Young +150 F train-clean-100 Fox in the Stars +151 F train-other-500 Gwen +152 M train-other-500 Andy Minter +153 M train-other-500 Graham Williams +154 M train-clean-360 Robert Foster +157 M train-clean-360 Ben Douglas +159 M train-clean-360 hugh mac +161 M train-other-500 Cyril Law, Jr. +163 M train-clean-100 Andrew Miller +166 F train-clean-360 Paula Berinstein +167 M train-other-500 Peter Yearsley +168 M train-other-500 Chris Goringe +173 F train-other-500 vlooi +174 M dev-clean Peter Eastman +175 F train-clean-360 Meredith Hughes +176 M train-clean-360 Vinny Bove +177 F train-other-500 Kymm Zuckert +188 F train-clean-360 Mary Anderson +192 F train-clean-360 Deborah Clark +196 M train-clean-100 Stewart Wills +198 F train-clean-100 Heather Barnett +199 F train-other-500 Maria Morabe +200 F train-clean-100 Maureen S. O'Brien +201 M train-clean-100 Joplin James +202 F train-other-500 Geetu Melwani +203 F train-clean-360 Marian Brown +204 M train-clean-360 Mark F. Smith +205 F train-clean-360 Esther +207 M train-clean-360 Kevin McAsh +208 F train-clean-360 Andrea L +209 F train-clean-360 Moira Fogarty +210 M train-clean-360 Aldark +211 F train-clean-100 shanda_w +215 F train-other-500 Alice Elizabeth Still +216 M train-clean-360 Thomas Davoren +217 F train-clean-360 firefly +218 M train-other-500 Kelly Clear +224 F train-clean-360 Caitlin Kelly +225 F train-clean-360 Brenda Dayne +226 F train-clean-100 Deb Bacon-Ziegler +227 F train-clean-360 Sarah Key-DeLyria +228 F train-other-500 Arctura +229 M train-clean-100 carnright +231 M train-clean-360 Sean McGaughey +233 M train-clean-100 Steve Karafit +237 F test-clean rachelellen +238 F train-other-500 Maria Elmvang +240 M train-clean-360 Shurtagal +242 F train-clean-360 Cagirlwithasoutherndrawl +243 F train-other-500 Jennifer Stearns +245 M train-other-500 Peter of Buckinghamshire England +246 F train-clean-360 Beth Dudek +248 F train-clean-100 Becky Miller +249 M train-clean-360 pww214 +250 F train-clean-100 Mary Reagan +251 M dev-clean Mark Nelson +252 M train-other-500 Rainer +253 M train-other-500 Stefan Schmelz +254 M train-clean-100 Alan Davis Drake (1945-2010) +255 M train-other-500 Chris Hawk +258 M train-clean-360 Kurt Wong +260 M test-clean Brad Bush +263 M train-other-500 Michael Sirois +264 M train-other-500 Chris Hughes +265 M train-other-500 Janice +272 M train-clean-360 Mr. Baby Man +273 M train-other-500 Jim Cadwell +274 F train-clean-360 Eileen George +277 F train-other-500 Alessia +278 F train-clean-360 AliceG +283 M train-other-500 Asaf Bartov +288 F train-clean-360 Bookworm +289 F train-clean-100 Barbara Wedge +294 F train-other-500 Calliope +296 M train-clean-360 Carl Manchester +298 F train-clean-100 Caroline Morse +302 F train-clean-100 Chris Peterson +303 M train-clean-360 Tony Hightower +307 M train-clean-100 Randy Phillips +310 F train-other-500 Dexnell Peters +311 M train-clean-100 deadwhitemales +313 F train-other-500 Dilini Jayasinghe +317 M train-other-500 Mike Gardom +318 F train-clean-360 Eileen aka e +319 M train-other-500 Ed Good +322 F train-clean-100 Elisabeth Shields +323 F train-clean-360 Erica Kuntz +328 F train-clean-100 Elizabeth Palmer +329 M train-clean-360 Todd Cranston-Cuebas +331 M train-other-500 Richard Grove +332 M train-clean-100 Aaron Teiser +335 M train-clean-360 Paul Harvey +336 M train-other-500 Harvey Chinn +337 F train-clean-360 Barbara Harvey +339 F train-clean-360 Heather Ordover +340 M train-clean-360 Scott Henkel +345 M train-clean-360 Micah Sheppard +348 M train-other-500 ianish +353 M train-clean-360 Jamey Osborne +359 M train-clean-360 John Nicholson +362 F train-clean-360 Judith Brown +365 M train-other-500 Jon Ingram +366 F train-other-500 Katy Preston +367 F test-other Kathleen Dang +369 M train-clean-360 Kevin Readdean +373 F train-clean-360 Kim Braun +374 M train-clean-100 kumarei +377 M train-other-500 Lenny Glionna Jr. +380 F train-clean-360 Laurie Campbell +392 F train-other-500 Maria Celano +398 M train-clean-360 James Smith +402 F train-other-500 Sharmini Kumar +403 F train-clean-100 Nocturna +404 F train-other-500 Nomenphile +405 M train-clean-100 Eric Dennison +408 F train-clean-360 Claudine Chen +409 M train-clean-360 Mike Kauffmann +412 M train-clean-100 Brian Roberg +413 M train-other-500 Daniel Watkins +421 M train-other-500 Patrick +422 M dev-clean President Lethe +426 F train-clean-100 Norah Piehl +428 M train-other-500 Rayburn Beale +432 M train-other-500 Steve Andersen +434 F train-clean-360 Joyce Nussbaum +439 M train-clean-360 Robert Garrison +441 F train-clean-100 Sandra in Wales, United Kingdom +444 F train-other-500 Sage Tyrtle +445 M train-clean-100 Dave Foss +446 M train-clean-100 Steve Hartzog +448 F train-other-500 scrappylibrarian +451 F train-clean-360 Sonserae Leese-Calver +453 M train-other-500 Glen Hallstrom +454 M train-clean-360 Tom Yates +458 M train-clean-100 Scott Splavec +459 M train-clean-360 Mark Bradford +460 M train-clean-100 Dave Ranson +464 M train-clean-360 Mike Wilson +466 F train-other-500 Joy Chan +470 M train-other-500 Chris Chapman +472 F train-clean-360 Tina Tilney +474 M train-other-500 Zachary Brewster-Geisz +475 M train-clean-360 Jason X. +476 M train-clean-360 Chuck Spann +479 F train-clean-360 wedschild +480 M train-clean-360 Chris Vee +481 M train-clean-100 Neal Foley +483 F train-other-500 junk science +487 M train-clean-360 Clayton J. Smith +489 F train-other-500 Tora +492 M train-clean-360 TBOL3 +497 M train-clean-360 audiotoshokan +500 M train-clean-360 galaxiant +501 M train-clean-360 mikenkat +505 M train-other-500 Menno +510 M train-clean-360 Kirk Thomas +511 M train-clean-360 Matthew Shepherd +512 M train-clean-360 Anthony Craine +517 M train-other-500 Matthew Walton +525 F train-clean-360 Victoria Long +533 F test-other Ana Simao +534 F train-clean-360 Jean O'Sullivan +542 M train-other-500 J. Hall +543 M train-clean-360 Ted Delorme +544 M train-other-500 bozgeez +548 M train-clean-360 Chris Mitchell +549 F train-clean-360 SarahHadley +551 M train-other-500 Guntar +557 M train-other-500 fieldsofgold +559 M train-clean-360 Bill Stackpole +561 M train-clean-360 Quentin +567 M train-other-500 Aaron Benedict +568 M train-other-500 JD Weber +569 M train-other-500 Frank +572 M train-other-500 Rebecca Dittman +576 F train-clean-360 Caroline Mercier +580 M train-clean-360 Ryan +581 M train-clean-360 C. Berrius +583 M train-clean-360 Russ Maxwell +584 F train-other-500 miette +585 F train-other-500 pheo +587 F train-clean-100 Joy Scaglione +589 F train-clean-360 Stephanie Konig +593 M train-clean-360 Eric S. Piotrowski +594 M train-clean-360 KentF +596 F train-clean-360 Carol Goode +597 F train-clean-360 Lisa Chau +598 F train-clean-360 Kim +606 M train-clean-360 Julian Jamison +608 F train-other-500 Baranduin +612 F train-clean-360 Cindy Steib +613 M train-other-500 D.E. Wittkower +614 F train-other-500 Christine Blachford +622 M train-other-500 Andrew Lebrun +625 M train-clean-100 toriasuncle +636 F train-clean-360 Zale Schafer (Rose May Chamberlin Memorial Foundat +637 M train-clean-360 Michael Scherer +639 M train-clean-360 Robert Beach +644 M train-other-500 Daniele +652 M dev-clean Scott Walter +663 M train-clean-360 Bruce Stafford +664 F train-clean-360 Wendy G. +666 F train-clean-360 Monique +667 F train-clean-360 Bethany Simpson +669 F train-clean-100 Anne +671 M train-clean-360 koijmonop +672 M test-clean Taylor Burton-Edward +679 M train-other-500 rhodian +681 F train-other-500 Lucy Burgoyne +684 F train-other-500 Lizzie Driver +688 F train-clean-360 J. M. Smallheer +690 F train-other-500 Silver +696 F train-clean-100 Tamara R. Schwartz +698 F train-clean-360 Randi Warwick +699 F train-clean-360 Diana Kiesners +700 F dev-other Susan Hooks +705 F train-other-500 eva +707 M train-clean-360 Jason Mayoff +708 M train-clean-360 Kevin Devine +711 M train-clean-360 Roy Schreiber +712 M train-other-500 Michael Shook +713 F train-other-500 Onjana Yawnghwe +716 M train-clean-360 martyd +718 M train-clean-360 clarknova +720 M train-other-500 Peter Gallagher +724 M train-clean-360 Michael Crowl +726 M train-other-500 Paul +727 M train-other-500 Andrew Richards +728 M train-other-500 Eric Connover +730 F train-clean-100 Karen Labenz +731 F train-clean-360 Megan-Jane Daniels Suyasu +737 M train-other-500 Roger W. Barnett +742 M train-other-500 Peter Groom +753 M train-other-500 Tim Bulkeley +764 M train-clean-360 Carl Vonnoh, III +766 M train-other-500 Jean Crevier +770 M train-clean-360 Justin S Barrett +774 F train-other-500 Sara Walsh +777 M dev-clean fling93 +778 M train-other-500 Branko Collin +779 M train-other-500 Greg +780 M train-other-500 Lloyd Davis +781 M train-clean-360 Mitchell Dwyer +782 F train-other-500 Michele Pacey +783 F train-clean-360 Gina +789 F train-other-500 Eva +791 M train-other-500 Ivan +792 F train-other-500 mjd-s +797 M train-other-500 Mike Shapiro +803 M train-clean-360 Greg Elmensdorp +806 M train-clean-360 Aaron Andradne +807 M train-other-500 Luke Venediger +810 M train-other-500 Joseph Loverti +811 F train-other-500 Elizabeth +815 M train-clean-360 mawrtea +816 M train-clean-360 Jeff Robinson +820 M train-clean-360 Scoot +826 M train-other-500 Thomas Wells +829 F train-clean-360 frenchfry +830 F train-clean-360 sebrazer +831 M train-clean-100 Nick Gallant +834 F train-clean-360 nausicaa +835 M train-clean-360 echo +836 M train-clean-360 Kevin LaVergne +839 M train-clean-100 rovert405 +844 F train-other-500 Martina +845 M train-other-500 DaveF +846 M train-other-500 Anadaxis_Canejia +850 M train-clean-360 tonypettit +851 M train-other-500 brenthumphries +868 M train-clean-360 Mike Rosenlof +876 F train-other-500 Alisha +882 F train-clean-360 Mur Lafferty +884 M train-other-500 sayeth +886 M train-other-500 Paul S. Jenkins +887 F train-clean-100 Lana Taylor +895 M train-other-500 Max Porter Zasada +899 M train-clean-360 thomahal +908 M test-clean Sam Stinson +909 M train-clean-100 Greg Bryant +911 M train-clean-100 frankjf +915 M train-other-500 Ted Kaouk +920 F train-clean-360 Sarah Bean +921 M train-other-500 Brian J. Callaghan +922 M train-clean-360 Steven H. Wilson +923 F train-other-500 Layna +925 F train-clean-360 pattymarie +927 M train-other-500 Nerijus +937 F train-other-500 Susie G. +948 F train-clean-360 Chere Theriot +949 M train-clean-360 ontheroad +951 F train-other-500 thomasina +953 M train-clean-360 Jim Mullins +954 M train-clean-360 Brooks Seveer +956 F train-other-500 krithiga +957 M train-clean-360 iscatel +960 F train-other-500 Marloes Schoonheim +964 M train-other-500 Paul Sze +968 F train-clean-360 Pat Elder +969 M train-other-500 Czechchris +976 F train-other-500 Alison Raouf +978 M train-other-500 Rick Box +979 F train-clean-360 Kelli Robinson +982 M train-other-500 Mike Roop +984 M train-clean-360 J A Carter +985 M train-other-500 George Pilling +986 M train-clean-360 Michael Kirkpatrick +1001 M train-clean-360 Eric +1006 F train-other-500 Marta Kornowska +1012 F train-clean-360 Lizzie Oldfather +1018 M train-clean-360 JimmyLogan +1025 M train-clean-360 rdmagpie +1027 M train-clean-360 Brooks Jensen +1028 M train-clean-360 Tim Lundeen +1031 F train-clean-360 swroot +1034 M train-clean-100 Kevin O'Coin +1040 M train-clean-100 John Garvin +1046 M train-clean-360 durnburr +1049 M train-other-500 Sam Fold +1050 F train-clean-360 entada +1051 F train-other-500 E. Moulton +1052 F train-clean-360 Kathy Jacobs +1053 F train-clean-360 katyleah +1054 F train-clean-360 Igor Teaforay +1058 M train-clean-360 James Tiley +1060 F train-clean-360 Val Grimm +1061 F train-clean-360 Missie +1065 M train-other-500 Justin Brett +1066 F train-clean-360 Laurie Anne Walden +1069 F train-clean-100 Dawn +1079 F train-clean-360 Mary aka Breadchick +1081 M train-clean-100 Fracture +1084 F train-other-500 Nichole Karl +1085 M train-other-500 hefyd +1088 F train-clean-100 Christabel +1089 M test-clean Peter Bobbe +1092 F train-other-500 Maria +1093 F train-clean-360 Kiki Baessell +1094 M train-other-500 tubeyes +1096 M train-other-500 Geoff Dugwyler +1097 M train-other-500 Euthymius +1098 F train-clean-100 Merryb +1100 F train-clean-360 Danielle Flores +1107 M train-other-500 Jason Oakley +1110 M train-other-500 Graeme Jolliffe +1112 M train-clean-360 RedToby +1116 F train-clean-100 Megan Stemm-Wade +1121 M train-clean-360 John Lieder +1124 F train-other-500 Ancilla +1132 M train-other-500 Giles Baker +1152 F train-other-500 Millbeach +1154 F train-other-500 Larysa Jaworski +1160 M train-clean-360 Gary Gilberd +1161 M train-other-500 Dominic Moore +1165 M train-clean-360 Bob Graff +1166 F train-other-500 Debra Lynn +1168 F train-other-500 Ree +1171 F train-other-500 Julia Claussen +1175 M train-clean-360 Brother Patrick +1179 M train-other-500 Alan Chant +1182 M train-clean-360 Brett Condron +1183 F train-clean-100 roolynninms +1184 M train-other-500 Jeremy Pavier +1187 M train-other-500 Paul Hansen +1188 M test-clean Duncan Murrell +1195 F train-clean-360 Jennette Selig +1200 M train-other-500 hosmer_angel +1212 F train-clean-360 Lee Ann Howlett +1221 F test-clean Dianne +1222 M train-clean-360 Joseph Ugoretz +1224 F train-clean-360 Heather Duncan +1225 M train-other-500 Chris Langston +1226 M train-clean-360 Russ Lemker +1230 M train-other-500 Ian Skillen +1235 M train-clean-100 Tim Gregory +1239 M train-other-500 Scott Robbins +1241 F train-clean-360 Catherine Fitz +1246 F train-clean-100 Sandra +1250 F train-other-500 Christina Boyles +1252 F train-other-500 Avery +1255 M dev-other Simon Evers +1258 M train-other-500 Mellors +1259 F train-clean-360 Elizabeth Klett +1260 M train-other-500 Chris Hughes +1261 M train-other-500 Mans Broo +1263 F train-clean-100 Leonie Rose +1264 M train-clean-360 Matthew Hinman +1265 M train-clean-360 Edward Elmer +1266 F train-other-500 Jenilee +1271 M train-clean-360 Christian Pecaut +1272 M dev-clean John Rose +1274 M train-other-500 Larry Gilman +1280 M train-other-500 Tim Makarios +1283 M train-clean-360 Paul Siegel +1284 F test-clean Daniel Anaya +1289 F train-clean-360 Joanne Pauwels +1290 F train-clean-360 librarianite +1291 F train-other-500 Patti Brugman +1296 F train-clean-360 Gigi Minden +1298 F train-other-500 Wina Hathaway +1311 M train-clean-360 Scott D. Farquhar +1313 M train-clean-360 Scott Sherris +1316 M train-clean-360 Estragon +1320 M test-clean number6 +1322 M train-clean-360 chris tierney +1323 M train-clean-360 Leon Mire +1331 M train-other-500 Adrian Praetzellis +1334 M train-clean-100 John Schell +1335 F train-clean-360 Clarica +1336 M train-clean-360 Charlie Blakemore +1337 M train-clean-360 Steven Rushing +1341 M train-other-500 Coastalbloke +1342 F train-other-500 SueAnn Dozier +1343 F train-clean-360 Laura Koskinen +1347 M train-other-500 Ted Nugent +1348 F train-clean-360 Janet Friday +1349 M train-clean-360 John Pruden +1353 M train-other-500 Clarke Bell +1355 M train-clean-100 Chris Gladis +1363 F train-clean-100 Tammy Sanders +1365 M train-clean-360 Joel Poortenga +1367 M train-other-500 Joe Brenneman +1370 F train-other-500 Lee Elliott +1373 F train-other-500 Kira Belkin +1374 M train-other-500 Graham Thomsen +1379 M train-clean-360 Ken Crooker +1382 F train-clean-360 Heather Lawrence +1383 M train-clean-360 David Best +1384 M train-other-500 Stephen Lamb +1387 M train-clean-360 Scott Mather +1390 F train-clean-360 C.L.Coney +1392 M train-clean-360 Chris Masterson +1401 F train-clean-360 Sibella Denton +1403 M train-other-500 tipaew +1413 M train-clean-360 ryanaw +1414 F train-other-500 guava +1417 F train-clean-360 Psuke Bariah +1421 F train-other-500 Madame Tusk +1422 F train-clean-360 Yazpistachio +1425 F train-clean-360 Jeanette Ferguson +1430 M train-other-500 Alok Karulkar +1444 M train-other-500 Ryan Mease +1445 M train-clean-360 Michael Yard +1446 M train-clean-360 Michael Loftus +1447 F train-clean-100 Luigina +1448 F train-clean-360 marevalo +1455 M train-clean-100 webslog +1456 M train-clean-360 Jason Isbell +1460 F train-clean-360 E. Plein +1462 F dev-clean E. Tavano +1463 F train-clean-360 Vivian Bush +1469 M train-other-500 Fr. Richard Zeile of Detroit +1472 F train-clean-360 Sarah Jennings +1473 M train-clean-360 Dan Polanco +1474 F train-other-500 Jc Guan +1482 M train-clean-360 Joshua B. Christensen +1485 M train-other-500 Robert Flach +1487 M train-clean-360 radioreader +1492 M train-other-500 mb +1494 M train-other-500 George Deprez, PhD +1495 M train-other-500 Glendower Jones +1498 F train-clean-360 Lori Hebel +1502 F train-clean-100 Ann Boyer +1505 M train-other-500 Mark Norman +1509 F train-clean-360 Miranda Stinson +1513 M train-clean-360 Simon-Peter Zak +1535 M train-clean-360 Robert Scott +1536 M train-clean-360 Marco +1544 F train-other-500 LilianaVale +1545 F train-other-500 AmyAG +1547 F train-clean-360 Riddleman +1552 M train-clean-360 Roger Turnau +1553 F train-clean-100 Mim Ritty +1556 M train-clean-360 geofred +1559 M train-other-500 Luke Harrison +1563 F train-other-500 Chandra Gioiello +1564 F train-other-500 Hedvig +1566 F train-other-500 Anna Christensen +1569 F train-other-500 Kristine Mackin +1571 M train-clean-360 Bob Tassinari +1572 M train-other-500 Alan Clare +1578 F train-clean-100 Lorelle Anderson +1579 F train-other-500 Philippa Willitts +1580 F test-clean TinyPines +1585 F dev-other Nelly () +1593 F train-other-500 Kristine Bekere +1594 M train-clean-100 Jon Scott Jones +1595 M train-other-500 Riccardo Fasol +1601 M train-other-500 Michael Yourshaw +1603 M train-clean-360 Arouet +1607 M train-clean-360 Claude Banta +1614 M train-other-500 FNH +1618 M train-other-500 Nicholas James Bridgewater +1621 M train-other-500 Caliban +1624 M train-clean-100 Daniel Shorten +1629 F train-clean-360 Gwyneth +1630 F dev-other spiritualbeing +1633 F train-other-500 Beecher +1634 M train-clean-360 daxm +1636 F train-other-500 Sandra Zera +1638 M train-clean-360 Kyle M. +1639 M train-clean-360 Joe Konno +1641 F train-clean-360 Rohanna +1643 M train-other-500 Chris Leslie-Hynan +1645 M train-clean-360 David Shamp +1646 M train-other-500 Ben Cobbett +1647 M train-other-500 Rich Meyers +1648 F train-other-500 Accent +1649 F train-clean-360 Kalynda +1650 M dev-other WangHaojie +1651 M dev-other Brendan Hodge +1653 F train-other-500 Carmina Sansone +1664 F train-other-500 Shauna M +1665 F train-other-500 Jessica AC Snyder +1668 F train-clean-360 stepheather +1673 F dev-clean Tonia +1674 F train-other-500 Jo +1678 F train-clean-360 leonardswench +1679 F train-other-500 Polly +1680 F train-other-500 intothelight +1681 F train-other-500 islajane +1685 M train-other-500 Jonathan Horniblow +1686 F dev-other neelma +1688 M test-other winam +1690 F train-other-500 Anne-Marie +1691 F train-other-500 Classicsfan +1693 F train-other-500 4Cullen +1695 F train-other-500 Steph +1696 F train-other-500 Darcywil +1699 M train-other-500 Gavin Smith +1701 M dev-other camelot2302 +1704 F train-other-500 JB +1705 F train-clean-360 tittletattle +1708 M train-other-500 Alaaious +1710 F train-other-500 Gilly +1714 F train-other-500 lauralee +1715 F train-other-500 Marianna +1717 F train-other-500 PJ +1721 F train-other-500 Linnea +1723 M train-clean-100 Rob Whelan +1724 F train-clean-360 Anna Simon +1726 F train-other-500 janeite +1731 F train-clean-360 Dani +1733 F train-other-500 Mira Cheskis +1734 F train-clean-360 LuvDemBrooders +1736 F train-other-500 LC +1737 F train-clean-100 Erin Hastings +1740 F train-clean-360 Shubda +1743 M train-clean-100 Bryan Ness +1746 M train-other-500 Theo Bacher +1748 M train-clean-360 Brad Powers +1750 F train-other-500 Lorie Heinrichs +1752 F train-clean-360 Jan MacGillivray +1754 F train-clean-360 Joan Freeman +1756 F train-other-500 Tamara Hamilton +1757 F train-other-500 cricket +1760 F train-other-500 Matthew Howell +1765 F train-other-500 Kelly Elizabeth +1767 F train-other-500 Cori Dean +1769 M train-clean-360 ej +1772 M train-other-500 David A. Stokely +1773 F train-other-500 Eliza Horne +1776 M train-clean-360 Jim Eastman +1777 M train-clean-360 Professor Chronotis +1779 F train-clean-360 Cynthia Zocca +1780 M train-other-500 Micah +1784 F train-other-500 grovejade +1789 M train-clean-360 Vin Reilly +1795 M train-other-500 Muhammad Mussnoon +1800 F train-clean-360 Scarlett! +1801 M train-clean-360 Antonio +1804 F train-other-500 Marie Manis +1806 M train-clean-360 Gary W. Sherwin +1809 F train-other-500 cucciasv +1811 M train-clean-360 Eric Ray +1813 F train-other-500 tesoro007 +1815 M train-other-500 Aringguth +1819 F train-other-500 Shannon +1825 F train-clean-360 srshel +1826 M train-clean-360 Jacob Miller +1827 M train-clean-360 Doug Wetzel +1828 M train-other-500 James Gladwin +1841 F train-clean-100 Laura Caldwell +1844 M train-other-500 noonday +1845 F train-clean-360 Katie Gibboney +1846 M train-other-500 valikojohn +1849 M train-clean-360 Kelly Dougherty +1851 F train-clean-360 Kehinde +1859 F train-clean-360 Jan Baxter +1863 F train-other-500 Ania +1867 M train-clean-100 Rowdy Delaney +1868 M train-other-500 Graham Redman +1870 M train-other-500 Stuart Bell +1874 M train-clean-360 Ernst Schnell +1878 M train-other-500 BLRossow +1885 F train-clean-360 inkwelldragon +1898 F train-clean-100 Jennifer +1901 F train-other-500 Allyson Hester +1903 M train-clean-360 Michael Thomas Robinson +1913 M train-clean-360 Geoff Cowgill +1914 M train-clean-360 Kevin Kivikko +1919 F dev-clean nprigoda +1920 F train-other-500 Annika Feilbach +1923 F train-clean-360 Maire Rhode +1924 M train-other-500 Andrew Drinkwater +1926 F train-clean-100 Nikki Sullivan +1931 F train-other-500 poormedea +1933 F train-clean-360 iremonger +1938 M train-other-500 icyjumbo (1964-2010) +1943 M train-clean-360 Corun +1944 F train-clean-360 Carolyn Frances +1958 M train-clean-360 Furio +1961 F train-clean-360 Qhali +1963 F train-clean-100 Belinda Brown +1968 F train-other-500 lizzyblack +1970 F train-clean-100 Dawn Larsen +1974 F train-clean-360 Katie Baynes +1977 F train-other-500 Jennie Hughes +1985 M train-other-500 Jonny Lee +1987 M train-clean-360 Michael Macedonia +1988 F dev-clean Ransom +1989 M train-other-500 Sergio Baldelli +1992 F train-clean-100 Michelle White +1993 F dev-clean Wendy Belcher +1995 F test-clean AJai Hilton +1998 F test-other Sonja +2001 M train-other-500 Phillip David +2002 M train-clean-100 Larry Maddocks +2003 M train-other-500 The Penang Lawyer +2004 F train-clean-360 Kim S +2007 F train-clean-100 Sheila Morton +2010 F train-clean-360 Julie Bynum +2012 M train-clean-360 jburby +2013 M train-other-500 Mark +2021 M train-other-500 Keri Ford +2026 F train-other-500 Mil Nicholson +2033 M test-other Filippo Gioachin +2035 F dev-clean Sharon Bautista +2039 M train-clean-360 Anton +2042 F train-other-500 Charlene V. Smith +2045 M train-clean-360 David O'Connell +2046 M train-other-500 kyleti +2050 F train-other-500 Xe Sands +2051 M train-other-500 Grant Petersen +2053 F train-clean-360 Vincent Tapia +2056 F train-clean-360 Nancy Roberts +2060 F train-clean-360 Julie Pandya +2061 F train-clean-360 Jodi Krangle +2062 F train-other-500 Mindy H +2063 M train-other-500 hearhis +2067 M train-other-500 Nick Gisburne +2068 F train-other-500 Priya, India +2074 M train-clean-360 Tysto +2078 M dev-clean Kathy Caver +2085 F train-clean-360 Stephanie Dupal-Demartin +2086 M dev-clean Nicodemus +2089 F train-other-500 Martina +2090 F train-other-500 Melissa +2092 F train-clean-100 Elaine Hamby +2093 M train-clean-360 RK Wilcox +2094 F test-clean amycsj +2096 M train-other-500 Nick Marsh +2100 F train-other-500 Katherine Holt +2104 M train-other-500 R. S. Steinberg +2110 M train-clean-360 Aaron Elliott +2113 M train-clean-360 Andrew Vidal +2122 M train-other-500 Kenneth R. Morefield +2127 M train-clean-360 wrongshore +2133 M train-other-500 Mat Messerschmidt +2136 M train-clean-100 Great Plains +2137 M train-clean-360 Jerome Lawsen +2140 M train-other-500 Ralph Snelson +2143 F train-other-500 Cat Schirf +2146 M train-clean-360 Jeff Stuckey +2148 F train-other-500 BethAnne +2149 M train-clean-360 Mark Penfold +2152 F train-other-500 redabrus +2156 M train-clean-360 Roger Melin +2159 M train-clean-100 Matthew Westra +2162 M train-clean-360 Ray Clare +2167 M train-clean-360 spiderman0521 +2182 F train-clean-100 Susan Umpleby +2185 M train-other-500 Jonathan Feldman +2194 F train-clean-360 RobbieRogers +2195 M train-other-500 Joe Earley +2196 F train-clean-100 Andrea Fiore +2198 M train-other-500 Clive Catterall +2201 M train-clean-360 Stephen Escalera +2204 F train-clean-360 Pamnache +2208 M train-other-500 Alan Brown +2229 M train-clean-360 Pete Williams, Pittsburgh, PA +2230 F train-clean-360 Isosceles +2234 M train-other-500 Lars Rolander +2237 F train-other-500 Chloey Winters +2238 M train-clean-360 Will Larson +2240 M train-clean-360 Ralph Volpi +2246 M train-other-500 RaySee +2254 F train-clean-360 Heidi Preuss +2256 F train-clean-360 tamurile +2262 M train-other-500 Andy +2269 F train-clean-360 Rhonda Federman +2270 F train-other-500 Megan Kunkel +2272 M train-clean-360 Alec Daitsman +2273 M train-other-500 Peter Kelleher +2275 F train-other-500 Lori H +2276 F train-other-500 Andrea +2277 F dev-clean zinniz +2279 M train-other-500 Gilles Lehoux +2284 M train-other-500 Zapo +2285 M train-clean-360 Bob Sage +2288 F train-other-500 Ellis Christoff +2289 M train-clean-100 David Kleparek +2292 M train-other-500 Dick Durette +2294 M train-clean-360 James Christopher +2297 F train-other-500 Philippa +2299 M train-clean-360 cpalmer17 +2300 M test-clean Mitchell L Leopard +2301 F train-other-500 Chris Jones +2309 M train-other-500 Wyatt +2312 F train-other-500 Lucy Lo Faro +2319 M train-clean-360 Jack Farrell +2334 M train-clean-360 David Lipa +2339 F train-other-500 skellie +2341 M train-other-500 webround +2346 F train-other-500 FirstKnight +2348 F train-clean-360 KellyLC +2351 F train-other-500 hugoceline +2356 M train-other-500 Paul Curran +2361 F train-other-500 M. J. Boyle +2364 F train-clean-360 Anna-Maria Viola +2368 M train-clean-360 Alex C. Telander +2374 F train-other-500 M.C.Y. +2380 M train-other-500 Jacob Cherry +2384 M train-clean-100 Ger +2388 M train-clean-360 Greg Bell +2391 F train-clean-100 treefingers +2393 M train-clean-360 Michael Bradford +2397 M train-clean-360 texttalker +2401 F train-clean-360 Matt Warzel +2404 M train-clean-360 n8evv +2405 M train-other-500 musil +2407 M train-other-500 ajmacbeth +2411 F train-clean-360 kristiface +2412 F dev-clean calystra +2414 M test-other Ashwin Jain +2416 F train-clean-100 Julia Albath +2427 M train-clean-360 Ed Meade +2428 M dev-clean Stephen Kinford +2436 M train-clean-100 Seth Adam Sher +2437 M train-other-500 Wetcoast +2445 F train-other-500 musici123 +2448 M train-other-500 David Federman +2473 M train-clean-360 Hoosemon +2481 F train-clean-360 Alana Jordan +2485 F train-other-500 Serin +2487 F train-other-500 Rachel Lintern +2488 F train-other-500 Lisa Wilson +2491 M train-other-500 johnb +2494 M train-clean-360 Mark Cawley +2496 M train-other-500 Ben Dutton +2498 F train-clean-360 B. Grebe +2499 M train-clean-360 Paul Henry Tremblay +2504 F train-other-500 Helen Elsbeth +2506 F dev-other Julie VW +2512 F train-clean-360 Marion +2514 M train-clean-100 S. Young +2517 M train-clean-360 Gayland Darnell +2518 M train-clean-100 Rob Powell +2522 F train-other-500 senshisteph +2526 M train-other-500 Bob Gilham +2531 M train-clean-360 Greg Weeks +2532 F train-clean-360 Jennifer Lott +2533 M train-clean-360 Steven Proctor +2541 M train-other-500 Eddie Winter +2544 F train-other-500 Annise +2545 M train-other-500 the quiet fox +2552 M train-other-500 Daniel Cranston +2553 F train-other-500 daisy55 +2562 M train-clean-360 Scott Merrill +2568 F train-other-500 Elena the Quiet +2570 F train-clean-360 kindlibrarian +2573 F train-clean-360 Becca B +2574 M train-other-500 TimSC +2577 F train-clean-360 K Hindall +2581 F train-clean-360 Julie Levi +2582 F train-clean-360 dolce +2587 F train-other-500 Anne Cheng +2588 M train-other-500 Padraig O'hIceadha +2589 M train-clean-360 Jordan +2592 F train-clean-360 Anna Roberts +2598 F train-clean-360 Barbara Bulkeley +2606 M train-other-500 Marc Tanti +2607 F train-other-500 Ruth Golding +2609 M test-other Ian Hatley +2618 M train-clean-360 Phil Surette +2624 M train-other-500 David Nicol +2625 F train-other-500 Auntie Em +2628 M train-clean-360 Zloot +2638 F train-clean-360 Dawn +2652 F train-clean-360 MixieArmadillo +2654 M train-clean-360 Notelrac +2660 M train-other-500 mpetranech +2671 F train-other-500 Foreign Girl +2673 M train-clean-360 jude kaider +2674 M train-clean-360 Jason Procopio +2676 F train-other-500 missizii +2688 M train-clean-360 Christopher Jennings +2691 F train-clean-100 Donna Stewart +2694 F train-other-500 DianaJMB +2696 M train-clean-360 anonymous +2709 F train-clean-360 PopularOutcast +2712 F train-other-500 anoldfashiongirl +2724 M train-other-500 Michael Dalling +2730 F train-other-500 Shirley Anderson +2733 M train-other-500 CalmDragon +2735 F train-other-500 ASchindler +2740 M train-other-500 Tony Ashworth +2741 F train-clean-360 Jan Dawn Doronila +2748 F train-other-500 Annoying Twit +2751 F train-clean-360 Angela Kelley +2754 F train-other-500 peaceuntoyou +2758 F train-clean-360 SopranoHarmony +2762 F train-other-500 Phillipa Chantry +2764 F train-clean-100 Piper Hale +2769 M train-clean-360 Anthony Wilson +2774 M train-clean-360 William Kevin Manire +2775 F train-clean-360 ink tree +2785 F train-clean-360 humeangel +2787 M train-clean-360 Quentin Manuel +2790 F train-clean-360 Victoria Slonosky +2792 F train-other-500 Varra Unreal +2803 M dev-clean aquielisunari +2812 M train-clean-360 Greg Hartley +2815 M train-clean-360 Michael Sample +2816 M train-clean-360 Andrew Symons +2817 F train-clean-100 Catherine Millward +2823 M train-clean-360 ChrisC +2825 M train-other-500 Ernst Pattynama +2827 M train-clean-360 David Leeson +2830 M test-clean Tim Perkins +2834 F train-other-500 Ksushi +2836 F train-clean-100 Linda McDaniel +2843 M train-clean-100 ricell +2853 F train-clean-360 Jane Greensmith +2854 M train-other-500 Andrew Coleman +2882 F train-clean-360 ayngelwing +2893 M train-clean-100 Ryan Sutter +2895 F train-other-500 Jenny Lundak +2902 M dev-clean dexter +2909 F train-other-500 Petra +2910 F train-clean-100 Janna +2911 M train-clean-100 David Lawrence +2919 F train-other-500 Raerity +2920 F train-clean-360 Rechelle +2925 F train-other-500 Darla +2929 M train-clean-360 Topaz +2930 M train-other-500 BUAES +2943 F train-other-500 Sarah Gutierrez +2946 M train-other-500 Markus Wachenheim +2952 M train-clean-100 Scott Carpenter +2960 M train-clean-360 Scotty +2961 F test-clean Leni +2967 F train-other-500 Landii +2971 M train-clean-360 Matthew C. Heckel +2975 F train-other-500 Sarafina Suransky +2979 F train-other-500 Jilliane Brandt +2985 M train-other-500 Cantor +2988 F train-other-500 Larissa Little +2989 F train-clean-100 Jamie Strassenburg, Cypress, California +2990 M train-other-500 Tom Crawford +2992 M train-clean-360 davechase +2997 F train-other-500 Violet +2998 M train-other-500 Kim Jansen +2999 M train-clean-360 Joseph Finkberg +3000 M dev-clean Brian von Dedenroth +3001 F train-clean-360 priscilla hewitt +3003 F train-clean-360 Sue Anderson +3005 M test-other Ken Tischler +3006 F train-other-500 MorganScorpion +3008 F train-clean-360 Gloria Zbilicki +3009 M train-clean-360 Jim Ruddy +3020 M train-other-500 Mike Vendetti +3021 M train-other-500 leetcr +3025 F train-clean-360 MichelleHarris +3032 M train-clean-360 Utek +3033 F train-other-500 valli +3045 M train-other-500 Cameron Conaway +3046 F train-clean-360 Stephanie Land +3053 F train-other-500 Tracy Yonemoto +3054 M train-other-500 GerryR +3060 M train-other-500 Didier +3063 M train-other-500 markman +3070 M train-clean-360 Mike Schwabe +3072 M train-clean-360 Chris Amos +3079 F train-other-500 nivedita +3080 F test-other breathe +3081 F dev-clean Renata +3082 M train-clean-360 Logan McCamon +3083 F train-clean-360 Emily Jarmard +3088 M train-other-500 Sean O'Hara +3090 F train-other-500 kelcymx +3092 F train-clean-360 Maggie Russell +3094 F train-clean-360 moe +3097 M train-other-500 nitram +3098 M train-other-500 woggy298 +3100 M train-other-500 David Higham +3105 M train-clean-360 gfairch511 +3109 M train-other-500 Diogenes Dog +3112 F train-clean-100 Jessica Louise +3114 M train-clean-360 Geoffrey Edwards +3118 M train-clean-360 George Yeager +3119 F train-clean-360 Sharon Riskedahl +3125 F train-other-500 Fran +3132 M train-other-500 Andy Yu +3135 M train-other-500 Steve Foreman +3137 M train-other-500 Parrot +3138 F train-other-500 Labyrinth Composer +3142 M train-other-500 RogerA +3143 F train-other-500 suzanne +3144 M train-other-500 jfmarchini +3148 M train-other-500 artos +3157 F train-clean-360 TriciaG +3168 M train-clean-100 David Anton +3170 M dev-clean VOICEGUY +3171 M train-clean-360 JoeD +3172 F train-other-500 Beatrice +3179 F train-other-500 Robin +3180 M train-clean-360 Mike Conrad +3185 M train-clean-360 JohnNewman +3187 M train-clean-360 Wasichu +3192 M train-other-500 Arfuhrm +3196 F train-other-500 Elizabeth Harker +3214 M train-clean-100 fourteatoo +3215 F train-clean-360 Shirley Ellen +3221 M train-clean-360 David Schoepf +3224 F train-clean-360 Acacia Wood +3227 F train-other-500 Betina +3228 F train-clean-360 Benuathanasia +3230 M train-clean-360 rasputin +3235 F train-clean-100 Karen Commins +3238 M train-other-500 nihilist00 +3240 M train-clean-100 flakker +3242 M train-clean-100 peac +3244 F train-other-500 wilwarin +3245 F train-other-500 wendy +3257 M train-other-500 Jay Vance +3258 F train-clean-360 mwalimu +3259 F train-clean-100 Kate West +3261 M train-other-500 fourgrays +3268 M train-other-500 Patrick McHaffie +3271 M train-other-500 Ancient mariner +3272 M train-other-500 Matthew_J_Almeida +3274 M train-clean-360 Morgan Saletta +3285 M train-other-500 David Collins +3288 M train-other-500 Euan Bayliss +3289 M train-clean-360 lukeprog +3290 F train-other-500 Marian Martin +3294 F train-clean-360 Marcy Fraser +3307 M train-clean-360 Doug Allison +3314 F train-other-500 Carol Stripling +3318 F train-other-500 Magdalena +3319 F train-other-500 mjbrichant +3328 M train-clean-360 Al Dano +3330 F train-clean-360 B. G. Oxford +3331 F test-other Peggy +3334 F train-other-500 joi +3340 M train-clean-360 Preston McConkie +3346 F train-other-500 Hannah Dowell +3347 F train-clean-360 tornadogrrrl +3356 F train-other-500 Diana Solomon +3357 F train-clean-360 swade +3361 F train-clean-360 Linda Lee Paquet +3368 M train-clean-360 Jim Allman +3370 M train-clean-360 Glenn Simonsen +3373 M train-other-500 cvd +3374 M train-clean-100 Craig Campbell +3379 F train-clean-360 Kele +3380 F train-clean-360 DrBeccaAnne +3381 M train-other-500 Cagliostro +3389 M train-clean-360 von +3394 F train-other-500 Jackie Provau +3400 F train-other-500 Laura Davis +3409 F train-other-500 Philippa Brodie +3411 F train-other-500 SuD +3417 M train-other-500 Albatross +3433 M train-other-500 Bob Sherman +3436 M train-clean-100 Anders Lankford +3440 F train-clean-100 Heidi Will +3446 F train-clean-360 kayo +3448 M train-clean-360 Todd Lennon +3465 F train-other-500 ravenotation +3467 F train-other-500 js392 +3470 M train-other-500 Jason Mills +3479 F train-other-500 Karan Yamada +3482 F train-clean-360 Hayden +3483 M train-clean-360 Alan Winterrowd +3486 M train-clean-100 Robin Balmer +3488 M train-other-500 Tom Weiss +3490 M train-clean-360 Gregg Margarite (1957-2012) +3493 F train-clean-360 Gail Mattern +3500 F train-other-500 B. Treadgold +3503 M train-other-500 Christian Al-Kadi +3513 F train-clean-360 Symmie +3521 M train-clean-360 NickNumber +3526 F train-clean-100 Bereni +3528 F test-other minimoose83 +3536 F dev-clean Arielle Lipshaw +3537 F train-clean-360 Tracy Datlen +3538 F test-other Wyndelyn +3540 M train-clean-360 Bill Ruhsam +3541 M train-other-500 Termin Dyan +3546 F train-clean-360 Jeannie +3547 M train-other-500 BenW +3549 F train-clean-360 Katie Riley +3551 F train-clean-360 Annie Kirkpatrick +3553 M train-other-500 Luc Kordas +3554 F train-other-500 LaraC, Louisville, KY +3557 F train-other-500 Rachel Triska +3559 F train-other-500 Kerry Hiles +3564 F train-other-500 Vanessa +3567 M train-other-500 David Lazarus +3570 F test-clean sarac +3571 M train-other-500 HarryInk +3575 F test-clean supergirl +3576 F dev-clean JudyGibson +3584 F train-clean-360 Bridget Gaige +3587 M train-other-500 John Nixon +3588 F train-other-500 Neeru Iyer +3592 M train-other-500 Paul McCartan +3595 M train-other-500 Martin Geeson +3598 F train-other-500 Dawn +3606 M train-other-500 Ashwath Ganesan +3607 M train-clean-100 Richard Wallis +3615 F train-clean-360 Lucy Perry +3618 M train-other-500 Timothy Ferguson +3630 F train-clean-360 Rachel Gatwood +3638 F train-clean-360 AmyG +3641 F train-other-500 Joelle Peebles +3645 F train-clean-360 MaryAnn +3647 F train-other-500 Channe +3650 M train-other-500 Jonathan Ross +3654 M train-clean-360 Mark Wilson +3656 M train-other-500 Kai Lu +3657 M train-other-500 Bellona Times +3660 M dev-other Russ Clough +3663 F dev-other ppezz +3664 M train-clean-100 Barry Eads +3665 M train-other-500 C.J. Casey +3675 F train-other-500 Linda Ferguson +3679 F train-other-500 veronasser +3681 F train-other-500 Ann Boulais +3686 M train-clean-360 Doug Delisle +3691 M train-other-500 Hollis Hanover +3698 F train-other-500 Nadine Eckert-Boulet +3699 M train-clean-100 Bruce Pirie +3703 F train-clean-360 Linda Andrus +3717 M train-clean-360 Elliott Miller +3723 M train-clean-100 Kevin Lavin +3728 F train-clean-360 amycs +3729 F test-clean Heather Hogan +3733 F train-clean-360 Melanie Schleeter McCalmont +3738 F train-clean-360 Collee McKinnon +3744 M train-other-500 Jonathan Burchard +3747 M train-other-500 Keith Henige +3752 M dev-clean Mark Welch +3757 F train-other-500 EmAllise +3764 F test-other Gabrielle Lambrick +3779 F train-other-500 Rachel Steely +3780 M train-other-500 Pete +3781 F train-clean-360 Celena Arter +3783 M train-other-500 TexasSteve +3790 F train-clean-360 hpark +3792 M train-clean-360 Brian Keith Barnes +3793 M train-other-500 Nathan +3796 M train-other-500 Mario Pineda +3798 F train-other-500 Bianca Kramer +3807 M train-clean-100 Jesse Noar +3816 F train-clean-360 Bev J Stevens +3819 M train-other-500 StarrDog +3825 M train-clean-360 Matt Wills +3830 M train-clean-100 rymd80 +3835 M train-clean-360 M.White +3843 F train-other-500 storm +3845 M train-other-500 Ray Smith +3848 F train-other-500 skoval +3851 F train-clean-360 mbousquet +3852 F train-clean-360 selniff +3853 F dev-clean M. Bertke +3857 M train-clean-100 Epistomolus +3864 M train-clean-360 Tom Watts +3866 M train-clean-360 SilverG +3867 F train-other-500 Roberta Carlisle +3869 M train-clean-360 Timothy Pinkham +3871 M train-other-500 Figura +3876 F train-clean-360 Frances Marcinkiewicz +3879 F train-clean-100 Keneva +3885 F train-other-500 Elli +3889 F train-clean-360 Alina +3894 M train-other-500 Indy Gosal +3895 M train-other-500 porlob +3896 M train-other-500 Will Zufall +3905 F train-clean-360 J. Rebecca Franklin +3906 F train-other-500 tabithat +3909 F train-other-500 Evelyn Clarke +3911 F train-other-500 Joseph Couves +3912 M train-other-500 Bob Neufeld +3914 M train-clean-360 Rob James +3915 F dev-other JenniferW +3922 F train-clean-360 Ashley Candland +3923 M train-clean-360 Sean Michael Hogan +3925 F train-other-500 Viglione +3926 F train-other-500 Denise Lacey +3927 M train-clean-360 Bob Stretch +3928 F train-other-500 Marianne Coleman-Hipkins +3934 F train-other-500 Lynne Carroll +3945 M train-clean-360 Floyd Wilde +3947 F train-clean-100 johnell +3955 M train-other-500 Fredrik Karlsson +3959 M train-other-500 rjhargrav +3962 M train-other-500 writerboyontour +3967 F train-clean-360 Christine Dufour +3969 F train-other-500 CM Slosson +3972 F train-clean-360 Joy Easton +3977 M train-clean-360 LivelyHive +3979 F train-clean-360 Dale A. Bade +3982 F train-clean-100 Kate Adams +3983 F train-clean-100 lavocedorata +3989 M train-clean-360 Rayne +3990 F train-other-500 Jessi +3992 F train-other-500 perpetualdreamworld +3994 F train-clean-360 Miriam Esther Goldman +3997 F test-other Sophia Choi +4005 F train-other-500 Jhiu +4009 F train-other-500 Diana Majlinger +4010 M train-clean-360 David Baldwin +4013 M train-clean-360 Kevin Maxson +4014 M train-clean-100 Tom Clifton +4015 M train-other-500 JimOCR +4017 M train-other-500 gsolgaard +4018 M train-clean-100 Nicholas Clifford +4019 M train-other-500 brrrrrr6 +4020 M train-other-500 Linda +4021 F train-other-500 Linda Woods +4034 F train-other-500 Sienna +4039 M train-clean-360 Shawn Craig Smith +4042 M train-other-500 Ryan DeRamos +4044 F train-clean-360 serenitylee +4051 F train-clean-100 Liz Devens +4054 M train-clean-360 Ryan Gubele +4057 F train-clean-360 RoseA +4059 M train-other-500 Troy Bond +4063 F train-other-500 Abigail Bartels +4064 F train-clean-360 Margaret Espaillat +4071 F train-clean-360 Nichelle von Lauder +4077 M test-clean Nathan Markham +4078 M train-other-500 Richard Kilmer +4085 M train-other-500 Paul P Miller +4088 F train-clean-100 Blazin48 +4090 F train-other-500 madmouth +4098 F train-clean-360 Rachell Lovett +4104 F train-other-500 Linda Dodge +4108 M train-clean-360 garymacf +4110 M train-clean-360 Richard Ellwood +4111 F train-clean-360 Rachel P. +4116 F train-clean-360 Amy Benton +4122 M train-other-500 Wendel Topper +4133 M train-clean-360 BigStory +4137 F train-clean-100 Sarah LuAnn +4138 F train-clean-360 daltongirl +4145 F train-clean-360 Patti Cunningham +4148 M train-clean-360 David A. Moore. +4152 M train-clean-360 Ata Khudayberdiev +4153 F dev-other Hilara +4156 F train-other-500 Caroline Shapiro +4160 F train-clean-100 Rosie +4161 M train-other-500 Daniel Paashaus +4172 M train-other-500 Graeme Dunlop +4174 F train-other-500 Availle +4179 M train-other-500 Robert Keiper +4189 F train-other-500 Megan Argo +4191 M train-other-500 Grant Hurlock +4192 F train-other-500 MaryModern +4193 M train-other-500 Ethan Rampton +4195 F train-clean-100 bj +4196 M train-other-500 Jim Clevenger +4198 M test-other Examinfo +4205 M train-other-500 psi_mon +4211 F train-other-500 A. Knight +4214 F train-clean-100 A. Janelle Risa +4216 F train-other-500 Hannah Skoonberg +4217 M train-other-500 Brendan Tannam +4218 M train-other-500 David Cole +4222 F train-clean-360 msjodi777 +4225 F train-other-500 gmiteva +4226 M train-clean-360 Jud Niven +4234 F train-other-500 laineyben +4235 F train-other-500 Haylayer Flaga +4236 M train-clean-360 Nicholas Feulner +4238 F train-clean-360 Chela +4243 M train-clean-360 Ian Grae +4246 M train-clean-360 JLaddJr +4257 M train-clean-360 garbageman99 +4260 F train-clean-360 Clacas +4262 F train-other-500 LoraBeth Davis +4263 F train-other-500 Chelsea Baker +4267 M train-clean-100 Ric F +4273 F train-other-500 as101 +4277 M train-other-500 JohanG +4278 M train-clean-360 smhamon +4280 F train-other-500 BB +4289 F train-clean-360 Veronica Jenkins +4290 M train-clean-360 Joshua Paul Johnson +4294 F test-other Jessamy Gloor +4295 M train-other-500 Slawek +4297 F train-clean-100 Tina Horning +4305 F train-other-500 Aspergine +4310 F train-other-500 grace4him +4313 F train-other-500 May Low +4321 F train-other-500 Xenutia +4323 F dev-other BookAngel7 +4327 F train-other-500 brokenaltar +4331 F train-clean-360 Barbara Clements +4335 M train-clean-360 davidb +4340 F train-clean-100 kiwafruit +4344 M train-other-500 Tadhg +4345 M train-other-500 Richard Schipper +4350 M test-other willem +4352 F train-other-500 Martina Hutchins +4356 F train-clean-360 Patricia Rutledge +4358 M train-clean-360 Kim Stich +4362 F train-clean-100 Michelle Montano +4363 F train-clean-360 Emily Livingston +4379 M train-other-500 Scott Dahlem +4381 F train-clean-360 Denise Resnik +4396 F train-other-500 Michael Wolf +4397 M train-clean-100 John Dennison +4402 M train-other-500 Christopher Sanner +4406 M train-clean-100 Matthew Scott Surprenant +4407 M train-other-500 Ted Drury +4411 F train-other-500 Zarnaz +4415 F train-other-500 Anita Fleming +4420 M train-other-500 Steve Lomas +4422 F train-other-500 Kamna +4423 M train-other-500 David Dwight +4425 M train-clean-360 Gary Coy +4427 F train-clean-360 Lynne Handler +4428 M train-other-500 Blueoyster101 +4433 M train-clean-360 Ken Sterry +4434 F train-clean-360 Sarah Nuxoll +4438 M train-clean-360 Greg W. +4441 M train-clean-100 William Peck +4442 M train-other-500 Pep +4443 F train-other-500 THOVO +4446 F test-clean Jen Maxwell +4447 M train-other-500 Cascades +4455 F train-other-500 Patrick Wells +4463 F train-other-500 Cate Mackenzie +4474 M train-other-500 John L. Clark +4480 M train-other-500 Iskander Shafikov +4481 F train-clean-100 margo zinberg +4484 M train-other-500 Phil Griffiths +4487 F train-other-500 Nienke +4490 F train-clean-360 Rachel Weaver +4492 F train-other-500 P.Hynes +4495 M train-clean-360 Dillon Stiles +4507 F test-clean Rachel Nelson-Smith +4511 F train-other-500 Jeanie +4513 F train-other-500 Gabi +4515 M dev-other Doug +4519 F train-clean-360 Mimi Wang +4520 F train-other-500 Dorlene Kaplan +4535 M train-clean-360 Brett W. Downey +4545 M train-other-500 SunshinePaul +4546 F train-other-500 Estelle Jobson +4549 F train-other-500 Kristen Zaza +4563 F train-other-500 Wiebke +4570 M dev-other Bill Mosley +4572 M dev-other om123 +4576 F train-other-500 Snaefaxi +4583 M train-other-500 Jersey City Frankie +4586 M train-clean-360 Chris Caron +4590 M train-clean-360 Diapadion +4591 M train-other-500 Robert White +4592 F train-clean-360 WestWestest +4594 M train-other-500 mailman61953 +4595 M train-clean-360 Eric Leach +4598 F train-clean-360 cher0520 +4599 M train-other-500 xibu +4629 M train-clean-360 Edward W. LaBonte +4640 F train-clean-100 Karen Mason +4652 F train-other-500 Savanna Herrold +4659 M train-other-500 Stephen Marsh +4660 F train-other-500 SusieSA +4667 F train-other-500 Maria Therese +4680 F train-clean-100 pachayes +4681 M train-clean-360 Frank Adams +4687 M train-other-500 mevans +4693 M train-other-500 Robert Parker +4697 F train-other-500 Jannie Meisberger +4699 M train-other-500 Jason Justice +4701 M train-other-500 CC +4703 F train-other-500 HurstPP +4705 M train-other-500 Algy Pug +4706 F train-other-500 SallyMc +4710 M train-other-500 David Huston +4712 F train-other-500 Alisson Veldhuis +4719 M train-clean-360 M. Craun +4731 F train-clean-360 Becky Cook +4733 M train-clean-360 dwegowy +4734 M train-clean-360 Adib Masumian +4738 M train-other-500 Leonard Wilson +4741 M train-other-500 Nullifidian +4742 F train-other-500 Little Tee +4744 F train-clean-360 Amy Gramour +4748 M train-other-500 Dirk Eichhorn +4750 M train-other-500 Paul Huckerby +4757 F train-other-500 Grace Dobson +4766 F train-other-500 Melanie +4767 M train-other-500 Delysid +4770 F train-clean-360 Jeanne Luft +4771 M train-other-500 scrawl +4773 F train-other-500 Kathryn Lois +4779 F train-other-500 Angel5 +4788 M train-clean-100 Bill Boerst +4791 F train-other-500 MelanieMae +4799 M train-other-500 Matt Judd +4800 F train-clean-360 Mary Herndon Bell +4806 M train-clean-360 Jason Ingolfsland +4807 F train-clean-360 Fuzz +4813 M train-clean-100 Steve Mattern +4821 M train-other-500 Zachary Johnson +4824 F train-other-500 meyerli +4830 M train-clean-100 George Aalto +4831 F dev-other Lisa Meyers +4836 M train-other-500 Preston Scrape +4837 F train-clean-360 Sharon Kilmer +4839 F train-clean-360 Julie K. Rose +4841 F train-other-500 Haley Pereira +4846 F train-clean-360 Kathrin Salazar +4848 M train-clean-360 Dee Wykoff +4852 M test-other Arthur Piantadosi +4853 F train-clean-100 Barbara Derksen +4854 M train-clean-360 Doctor_wu +4856 M train-clean-360 Steven Seitel +4859 M train-clean-100 nathank +4860 M train-clean-360 Jonah Cummings +4863 M train-other-500 I M CLIFFORD +4872 M train-other-500 R E Faust +4894 M train-other-500 Gary Dzierlenga +4898 M train-clean-100 greatbasinrain +4899 F train-clean-360 Theresa L. Downey +4910 F train-other-500 Amanda Martin Sandino +4915 M train-other-500 Andrew Bowles +4926 F train-clean-360 browneyedgirl32382 +4930 M train-other-500 Adrian Levitsky +4931 M train-other-500 KevS +4936 F train-other-500 jedopi +4945 M train-clean-360 Charles RUHE +4948 M train-other-500 Tom Barron +4955 M train-other-500 Elwood Mott +4957 F train-clean-360 P Moscato +4958 M train-other-500 David P. Sroka +4959 F train-other-500 Piper Hayes +4964 F train-other-500 rashada +4965 F train-other-500 Rachel Craig +4967 F train-clean-360 Karen +4969 M train-other-500 Garth Burton +4970 F test-clean airandwaters +4973 M train-clean-360 Michael Lipschultz +4979 M train-other-500 Eric +4991 F train-other-500 Jacqueline (Jacqui) Grady +4992 F test-clean Joyce Martin +4993 M train-other-500 BillMac +5000 F train-other-500 Anna-Lisa Ott +5002 M train-clean-360 Brendan Stallard +5005 F train-other-500 pinhsien +5007 F train-clean-360 Kathleen Nelson +5009 M train-other-500 Michael Reuss +5012 F train-clean-360 Morgan Schlicker +5013 M train-other-500 Joseph Dsouza +5019 F train-other-500 Sandra Estenson +5022 F train-clean-100 Kathleen Costa +5023 F train-other-500 Elizabeth Zaranka +5029 F train-clean-360 Courtney Sandhu +5036 F train-other-500 NicolaRuth +5038 M train-other-500 John Kooz +5039 F train-clean-360 Inga Parsons +5043 M train-other-500 edbucks +5044 F train-other-500 Andrea Keene +5045 F train-other-500 Mounica +5049 M train-clean-100 Bradley Smith +5054 F train-clean-360 Denice Stradling +5060 F train-other-500 Linda Moreau +5062 F train-clean-360 E Ogston +5063 F train-clean-360 susanfrom +5076 M train-other-500 Jules Hawryluk +5077 F train-other-500 Sandra G +5082 F train-other-500 SummerWind +5092 M train-clean-360 Josh Smith +5093 F train-clean-360 Nicole Kay +5101 F train-other-500 E.Lee +5104 M train-clean-100 Chuck Burke +5105 M test-clean elongman +5115 F train-clean-360 Terry Goodyer +5118 M train-other-500 klbonds +5123 M train-clean-360 wvthcomp +5126 M train-clean-360 Tom Lennon +5132 M train-other-500 David Stryker +5133 F train-clean-360 Jo Karabasz +5136 F train-other-500 Felicity C +5139 F train-clean-360 Katine +5141 M train-other-500 David Goldfarb +5142 F test-clean Mary Ballard-Johansson +5147 M train-clean-360 Cody2 +5152 M train-other-500 Liam Neely +5154 F train-clean-360 Danarra +5157 M train-clean-360 drewmac +5163 F train-clean-100 LilyAnne +5164 F train-other-500 Vsilverlining +5172 M train-other-500 Equilibrium33 +5181 M train-other-500 Joel Nisbet +5183 M train-other-500 Kenneth Sergeant Gaghan +5185 M train-other-500 okei +5186 M train-clean-360 Brendan Brown +5189 M train-clean-360 Samanem +5190 M train-clean-360 Rom Maczka +5192 M train-clean-100 Jason Esteves +5198 M train-other-500 Brian +5199 F train-other-500 Diana Fast +5206 M train-clean-360 Simon Dexter +5217 F train-other-500 Anqi Wang +5220 F train-other-500 Mary Schneider +5224 M train-other-500 Matt Soar +5230 F train-other-500 Aubrey Anne +5233 F train-other-500 summerdaze +5239 M train-clean-360 compozr +5242 M train-clean-360 Michael Monhollon +5244 F train-other-500 Carolin Ksr +5245 M train-other-500 Dymmesdale +5246 M train-clean-360 Mike Bloomfield +5248 M train-other-500 Don Stirno +5252 M train-other-500 Michael Deng +5261 M train-clean-360 Ali Kazerani +5266 F train-clean-360 Jill +5269 F train-other-500 Elizabeth Barr +5271 F train-other-500 Amanda +5278 M train-other-500 Mike Harris +5280 M train-other-500 penboy7000 +5285 F train-other-500 Sweetlilbirdy +5287 F train-other-500 adsum iam +5290 F train-clean-360 Natalie +5293 M train-clean-360 Ned Troxel +5296 F train-other-500 Joselyn Hasty +5299 F train-other-500 Becky Doughty +5304 M train-clean-360 Don W. Jenkins +5319 M train-clean-360 Guero +5321 M train-other-500 Adam Whybray +5322 M train-clean-100 Jay Bidal +5325 M train-other-500 Rory Lawton +5328 M train-other-500 Nathan Jordan +5333 M train-clean-360 Robert Fletcher +5337 F train-clean-360 Danielle +5338 F dev-clean S R Colon +5339 F train-clean-100 Lauren McCullough +5340 F train-other-500 Mary-Beth Blackburn +5350 F train-other-500 Cath Garde +5355 F train-other-500 Carrie Heyes +5361 F train-other-500 scarlettraces +5375 F train-other-500 BumbleVee +5379 M train-other-500 lennich +5386 M train-clean-360 Tim Ferreira +5389 M train-clean-360 Joseph Lawler +5390 M train-clean-100 Charles Bice +5393 F train-clean-100 Amy Hengst +5400 F train-clean-360 Natalie Sullivan +5401 M train-clean-360 Andrew Nelson +5405 F train-other-500 Anka +5412 F train-other-500 tommybascue +5424 F train-other-500 Julia Niedermaier +5429 M train-other-500 Dennis Lane Pretoria +5439 F train-other-500 Laine S. +5442 F test-other oneiros81 +5445 F train-other-500 Liz DeLassus +5448 M train-clean-360 Bryan Reid +5456 M train-clean-100 e_scarab +5459 M train-other-500 bryan.peterson +5460 M train-other-500 Max Lindberg +5463 M train-clean-100 GLM +5468 F train-other-500 ashleighjane +5471 F train-other-500 ElleyKat +5480 F train-other-500 ESFJ Girl +5484 F test-other Nastassia +5487 F train-other-500 Michaela O'Connor +5489 F train-clean-360 LizMourant +5506 M train-other-500 Frank Booker +5513 M train-clean-360 David Callahan +5514 F train-clean-100 Ella Jane Quentin +5519 M train-clean-360 scoutman77 +5536 M dev-clean David Mix +5538 F train-clean-360 Linda Velwest +5543 F dev-other Anya +5545 M train-other-500 Gary Olman +5561 F train-clean-100 Ellen Jones +5565 M train-other-500 Ken Padgitt +5567 M train-other-500 BStapley +5569 F train-other-500 Beth Hitesman +5570 F train-clean-360 Corinna Schultz +5583 F train-clean-360 Claudia Wilson +5588 F train-clean-360 Gail Timmerman Vaughan +5604 F train-clean-360 kittyandcheese +5606 F train-clean-360 PrincessG +5618 F train-clean-360 Shirleyroses +5620 M train-other-500 jerryB +5622 F train-clean-360 Roseanne Schmidt +5628 M train-other-500 Adrian Wheal +5635 M train-clean-360 Matthew Reece +5636 F train-other-500 Joyce Couch +5637 M train-clean-360 Anthony +5639 M test-clean Ulf Bjorklund +5641 M train-other-500 Gerald Peter Morgan +5649 F train-other-500 hazelra +5652 F train-clean-100 amicrazy2u +5653 F train-other-500 Kimberly Anderson +5655 F train-clean-360 Christine Rodriguez +5656 M train-clean-360 Joe Mabry +5660 F train-clean-360 hollz +5661 F train-other-500 Dianne Lanning +5665 F train-other-500 Lynda Sizemore +5671 F train-other-500 Trihypoo +5672 M train-clean-360 Jacob Paul Starr +5678 M train-clean-100 jgoffena +5682 F train-other-500 Lmwong +5683 F test-clean Rachael Lapidis +5684 F train-clean-360 Elsa Youngsteadt +5688 F train-clean-100 Jennifer Dionne +5694 M dev-clean Winston Tharp +5700 M train-other-500 Paul Adams +5703 M train-clean-100 Garth Comira +5712 F train-clean-360 Alexandra Huckabay +5717 M train-clean-360 Phil Chenevert +5719 M train-other-500 John Fricker +5720 F train-other-500 Thelma Meyer +5723 F train-clean-360 Linda Hogan +5724 F train-clean-360 paintgirl +5725 F train-other-500 doublemirrors +5727 M train-clean-360 JDavidMoore +5731 F train-clean-360 Andrea Boltz +5733 F train-other-500 Spinhop +5735 F train-other-500 Filipa +5740 F train-clean-360 Alisa +5746 M train-clean-360 DL Pead +5750 M train-clean-100 laurencetrask +5756 F train-other-500 Mich_elle +5764 M test-other Herman Roskams +5765 M train-other-500 A. J. Carroll +5767 M train-clean-360 taijohn +5772 M train-other-500 jessecoy +5776 M train-clean-360 Chris Pauley +5778 F train-clean-100 Laura Victoria +5781 M train-other-500 Parrot17 +5784 M train-other-500 Chris Donnelly +5789 F train-clean-100 Kirsten Wever +5791 M train-other-500 croudy +5796 F train-other-500 Linette Geisel +5802 M train-clean-360 Christopher Maust +5808 M train-clean-100 jeandelfrio +5809 F train-clean-360 kattekliek +5810 M train-clean-360 Patrick Reinhart +5825 F train-other-500 Vira Denton +5826 M train-other-500 Phineas Redux +5831 F train-other-500 Kirsty Leishman +5837 F train-other-500 Debbie Pieterse +5840 M train-other-500 Dick Summerfield +5849 F dev-other ashleyspence +5854 M train-other-500 Gabriel Glenn +5860 F train-other-500 J L Raimundo +5867 F train-clean-100 Sharon Omi +5868 M train-clean-360 alwpoe +5874 M train-other-500 Paul Andrews +5876 F train-clean-360 kelleywyskiel +5883 M train-clean-360 Jerry Romero +5886 M train-other-500 Simon Brown +5890 F train-other-500 GabrielleC +5893 F train-other-500 Elizabeth Fiedler +5894 M train-other-500 Paul Stephens +5895 F dev-clean iamartin +5906 F train-other-500 Lily-LLM +5909 M train-clean-360 Mark Mickelson +5910 M train-other-500 Anthony Lee +5911 M train-other-500 Paul C. Newman +5913 M train-other-500 dmbrought +5914 M train-clean-360 bobbybrill +5918 M train-clean-360 Mike Wajda +5929 F train-other-500 Gryphon Perkins +5933 M train-other-500 Barry O'Neill +5935 M train-clean-360 Mike Okonek +5940 M train-clean-360 Gargoyle +5949 F train-other-500 debolee +5951 M train-other-500 Stevan Simmons +5952 F train-other-500 sherlock85 +5968 F train-clean-360 Cate Barratt +5970 M train-other-500 Larry Degala +5975 M train-clean-360 Tony Maxey +5977 M train-other-500 Ken Felt +5979 M train-other-500 Pentti Hirvonen +5980 F train-other-500 Nichole Thompson +5983 F train-other-500 Katew +5984 F train-clean-360 sbburke +5985 M train-clean-360 James K. White +5993 M train-other-500 garyday +6000 F train-clean-100 MissRose +6003 M train-other-500 sparks0314 +6006 F train-clean-360 Stephanie Lee +6009 M train-other-500 AVG +6010 M train-other-500 Ralph Kerwin +6014 F train-clean-360 Tina Nuzzi +6019 M train-clean-100 DerekP +6025 F train-other-500 Lonelle Yoder +6030 F train-other-500 Shiromi +6032 F train-clean-360 Beverly Scott +6035 M train-other-500 Marty Kris +6037 M train-clean-360 bish +6038 F train-clean-360 Sirmelja +6051 M train-other-500 Ric Cornwall +6054 F train-clean-360 Leslie Stevens Suhy +6060 M train-clean-360 Jonathan Lange +6064 F train-clean-100 Deborah Knight +6065 M train-other-500 David Olson +6070 F test-other Saab +6072 F train-other-500 CrowGirl +6075 M train-clean-360 Rob Smith +6076 M train-other-500 Saethram +6077 M train-other-500 Steve Belleguelle +6078 F train-clean-100 dobsonfly +6080 M train-clean-360 progressingamerica +6081 M train-clean-100 Lazuli +6082 F train-clean-360 EyeBones +6084 M train-other-500 Amallen +6087 F train-other-500 Caroline Driggs +6088 F train-other-500 burk +6097 M train-other-500 Phil Benson +6098 M train-clean-360 Kancamagus +6099 F train-clean-360 Cheri Gardner +6102 F train-other-500 Rachel Bossier +6104 F train-clean-360 Juliana M. +6106 F train-other-500 Ruth Kidson +6111 F train-other-500 Jill Preston +6115 M train-clean-360 MitchHerd +6119 M train-clean-360 Vinnie Tesla +6120 F train-clean-360 Terra Mendoza +6121 F train-other-500 Imagine +6123 F dev-other chocmuse +6126 M train-other-500 David Abbott +6127 F train-other-500 Andee +6128 F test-other Isabelle Brasme +6131 F train-other-500 Malane +6135 F train-other-500 Spike Holcomb +6138 F train-other-500 Deborah Brabyn +6139 F train-clean-360 Lois C. Johnson +6145 F train-other-500 Madeleine Brook +6147 F train-clean-100 Liberty Stump +6153 F train-other-500 anjieliu +6157 M train-clean-360 HotConflict +6159 M train-other-500 Ted Garvin +6160 F train-clean-360 FaithR +6167 F train-clean-360 Janet +6173 F train-other-500 Carol Box +6177 M train-other-500 wminbru +6178 F train-other-500 alegriavida +6181 M train-clean-100 Mike +6184 M train-other-500 Adam Tomkins +6188 F train-clean-360 Roxanna Nazari +6189 F train-clean-360 Shana Cohen +6196 M train-other-500 Martin Gradwell +6199 M train-other-500 bobolink +6206 F train-clean-360 Yvonne Smith +6209 M train-clean-100 deckerteach +6211 F train-other-500 Caeliveres +6215 F train-clean-360 Janet248 +6221 M train-other-500 pwu909 +6224 M train-other-500 dave k +6227 F train-other-500 EliMarieHK +6232 M train-other-500 Paul Denton +6233 F train-clean-360 Gen Jones +6235 F train-clean-360 ReadWriteLib +6236 M train-other-500 jcwyatt +6241 M dev-clean badey +6242 F train-other-500 Lucinda Gainey +6248 M train-other-500 Kevin Green +6249 M train-other-500 Thomas A. Copeland +6251 F train-other-500 Theresa Sheridan +6254 F train-other-500 Chelsea S. +6258 M train-clean-360 haggisreflux +6267 M dev-other Cata +6269 M train-clean-360 Don Halpert +6272 F train-clean-100 jlenardon +6276 M train-other-500 dan_h +6281 F train-other-500 Beth Thomas +6284 F train-other-500 cklee +6286 F train-clean-360 Wendy Almeida +6288 F train-clean-360 Sunni West +6294 M train-clean-360 humanode +6295 M dev-clean Michael Packard +6300 F train-clean-360 Sarah Crampton +6308 F train-clean-360 Easton +6311 M train-other-500 TRUEBRIT +6313 F dev-clean Jennifer Wiginton +6317 F train-clean-360 Sarah K +6319 F dev-clean thestorygirl +6323 M train-other-500 Caluminium +6324 F train-other-500 Barbara E. McCarthy +6330 M train-clean-360 David Cummings +6332 F train-other-500 Kelli England +6333 M train-other-500 Max Korlinge +6339 F train-clean-360 Michelle Remington +6341 F train-clean-360 Tiffany J. Kim +6345 F dev-clean Jean Bascom +6346 M train-other-500 Matthew Pagan +6351 F train-other-500 C. L. W. Rollins +6352 F train-clean-360 stmacduff +6353 F train-other-500 Jennifer Randall +6356 M train-other-500 John O +6358 F train-other-500 selway +6359 F train-clean-360 Susan Hanfield +6364 M train-other-500 Todd Ulbrich +6367 M train-clean-100 Vince Dee +6368 M train-other-500 Marc Pizzuti +6370 M train-other-500 Dan Craig +6371 F train-clean-360 Rebecca King +6373 F train-clean-360 Sandie Guenther +6377 M train-other-500 Maxim Babich +6378 F train-clean-360 Michelle Day +6385 F train-clean-100 Novella Serena +6388 F train-clean-360 Leda +6391 F train-other-500 polkadotish +6395 M train-clean-360 Richard Carpenter +6399 M train-other-500 Edmund Bloxam +6402 M train-other-500 Chiquito Crasto +6406 F train-clean-360 Abigail Rasmussen +6407 M train-other-500 AdamH +6411 F train-other-500 Feyaza +6415 F train-clean-100 Daryl Wor +6418 M train-other-500 Iolo Jones +6426 F train-clean-360 Pat Redstone +6432 M test-other Ditchdog +6436 M train-other-500 Steve Mattingly +6437 M train-clean-100 John Hoerr +6446 M train-clean-360 Bob Gonzalez +6454 M train-clean-100 David Wales +6455 F dev-other Betty Chen +6458 M train-clean-360 Dennis Blake +6459 M train-other-500 Nigel Boydell +6467 M dev-other sid +6476 F train-clean-100 Viridian +6482 M train-other-500 Delmar H Dolbier +6484 F train-other-500 ilianthe +6488 F train-other-500 Kendall Ashyby +6492 M train-clean-360 Hugh Gillis +6494 M train-clean-360 Morey Kunin +6497 M train-clean-360 James E. Carson +6499 M train-clean-360 dave7 +6505 F train-clean-360 Elena +6506 M train-other-500 Tim Quinn +6509 F train-clean-360 Marianna Foos +6510 M train-clean-360 JimD +6512 M train-other-500 mcgovern1934 +6513 F train-other-500 Susan de Raadt +6518 F train-clean-360 Wayfarer +6519 F train-clean-360 Monica Knuppe +6529 M train-clean-100 Fred DeBerardinis +6531 F train-clean-100 janesandberg +6533 M train-other-500 drewmore +6534 F train-other-500 Helen Falconer +6535 M train-other-500 Ron Altman +6538 M train-clean-360 Juan Federico +6539 M train-other-500 David W. Wolfe +6540 M train-other-500 Craig Gulliver +6544 F train-clean-360 Amanda Friday +6548 F train-other-500 Kristin Gjerlw +6549 F train-other-500 Sandra Luna +6550 M train-clean-360 elfpen +6553 M train-clean-360 DublinGothic +6555 M train-clean-360 Alexandre Laplante +6557 F train-other-500 Tika Sabu +6563 M train-clean-100 William Tomcho +6567 F train-clean-360 vikvenom +6568 F train-other-500 Heather Hamtil +6574 M train-clean-360 Caden Vaughn Clegg +6575 M train-clean-360 Patrick Painter +6583 F train-other-500 NastassiaS +6590 F train-other-500 Marea Brook +6594 F train-other-500 Amelia Chantarotwong +6599 M dev-other rohde +6609 M train-other-500 Patrick Wallace +6610 F train-other-500 sylly +6614 F train-other-500 Veronica Schlette +6620 F train-clean-360 Amy Koenig +6625 M train-other-500 hearmeout7 +6627 F train-other-500 Amber Hamilton +6636 M train-other-500 Philip Martin +6637 F train-clean-360 Christine Nendza +6641 F train-other-500 Julienne +6643 F train-clean-360 cbooren +6652 M train-other-500 Brendon Wright +6660 F train-other-500 Nicole Lee +6668 M train-other-500 Ecological Humanist +6670 M train-other-500 Mike Pelton +6673 F train-clean-360 Jenna Lanman +6674 M train-other-500 jimmylee +6676 F train-other-500 Chill28 +6683 F train-clean-360 Julia Kelley +6685 M train-other-500 Jonathan Drury +6686 M train-clean-360 Elin +6687 M train-other-500 KirksVoice +6689 F train-other-500 Tiffany Halla Colonna +6690 M train-clean-360 T Michael Burke +6694 M train-clean-360 Ross Williamson +6695 F train-other-500 Linda Fredericks +6696 F train-clean-360 Katryn Wiese +6701 M train-clean-360 Dayle +6705 F train-other-500 D. A. Frank +6707 M train-other-500 Lewis +6709 M train-other-500 vinphizz +6713 M train-other-500 stephenreader +6724 F train-other-500 Kristin Young +6726 F train-other-500 metgmz +6727 M train-clean-360 Tony Russell +6733 F train-other-500 Snapdragon +6735 F train-other-500 fshort +6741 F train-other-500 Sharon C. +6743 M train-other-500 Rick Rodney +6746 M train-other-500 Robin Skelcey +6747 M train-other-500 Ryan Lothian +6749 F train-other-500 MaryA +6752 M train-other-500 maxvon_d +6753 M train-other-500 T.E. McHenry +6754 M train-other-500 ToddHW +6758 F train-other-500 The Gypsy +6763 F train-clean-360 Manjit Bains +6773 M train-other-500 MostafaRazavi +6777 M train-other-500 Rick Saffery +6782 F train-clean-360 zcameo +6784 M train-other-500 SteveBuys +6788 F train-clean-360 Pamela Krantz +6792 M train-other-500 montmorency +6794 F train-other-500 Rachel Moyar +6798 M train-other-500 Aesthete's Readings +6804 M train-other-500 Nick Duncan +6807 F train-other-500 Lisa Caputo +6818 F train-clean-100 beckyboyd +6821 F train-other-500 Rholdah +6828 F train-clean-360 Lori Fuller Chugiak, AK +6829 F test-clean LadyBug +6836 M train-clean-100 John +6841 M dev-other A. E. Maroney +6846 M train-other-500 John Leonard +6848 M train-clean-100 KarlHenning +6849 M train-other-500 Dan Raynham +6853 F train-other-500 J. McKnight +6865 F train-clean-360 Jing Li +6875 M train-other-500 Bill Miller +6877 M train-clean-360 Bear Schacht +6880 M train-clean-100 Capybara +6882 M train-other-500 David Isenhower +6883 M train-other-500 Adam Doughty +6892 M train-other-500 Piotr Nater +6895 F train-clean-360 Reeses118 +6902 F train-other-500 Barbara Edelman +6904 F train-clean-360 Kirsten Nelson +6906 F train-other-500 Joanna1 +6912 M train-other-500 Richard Beck +6913 F train-other-500 daisyb +6914 F train-other-500 Katalina Watt +6918 F train-clean-360 Marilyn Mack +6923 M train-other-500 Szindbad +6924 F train-clean-360 Rapunzelina +6925 M train-clean-100 Thomas Meaney +6927 F train-clean-360 Sarika Pawar +6930 M test-clean Nolan Fout +6937 F train-clean-360 DVoice +6938 F test-other Simmy +6943 F train-other-500 Chieko Steely +6945 M train-other-500 Daniel George +6947 F train-other-500 Grace +6950 F train-other-500 elmay +6951 F train-other-500 redhed3095 +6954 M train-other-500 Paul Richards +6956 M train-clean-360 DannyHauger +6962 M train-other-500 mattoscarlomas +6963 M train-other-500 Kasper +6965 M train-clean-360 NoelBadrian +6967 F train-other-500 sganatra81 +6974 M train-other-500 Michael Armenta +6978 F train-other-500 Debra +6981 M train-clean-360 nlonghu +6993 M train-clean-360 Seyed +7000 M train-clean-360 Kevin Alix +7001 F train-other-500 Arienne +7008 M train-other-500 John Trevithick +7009 M train-other-500 roeg11 +7011 M train-clean-360 Icprice +7012 F train-other-500 Erin McKelle +7018 M test-other FSharp +7021 M test-clean Nodo420 +7026 F train-other-500 Michele Eaton +7030 M train-clean-360 Conrad T +7046 M train-other-500 Pascal Ramseier +7051 M train-clean-360 Andrew White +7055 M train-other-500 gemtwist +7059 F train-clean-100 Joannemmp +7061 M train-clean-360 AllenJohns +7062 F train-other-500 Rebecca Thomas +7065 M train-other-500 smitaj +7067 M train-clean-100 Matthew Wall +7069 M train-clean-360 John Schuurman +7073 F train-other-500 Jill Janovetz +7078 F train-clean-100 Mary in Arkansas +7079 F train-other-500 Chuck Williamson +7085 M train-clean-360 voicebynatalie +7090 M train-clean-360 Jon Sindell +7092 F train-other-500 Lorraine B +7095 M train-clean-360 Wesseling +7096 M train-other-500 Gary Iredale +7097 F train-other-500 novelreader +7105 M test-other jennycbnyn +7107 M train-other-500 titankin77 +7113 F train-clean-100 Sukaina Jaffer +7117 F train-clean-360 Art Leung +7120 F train-clean-360 L D Hamilton +7121 M train-other-500 384403 +7125 F train-other-500 Peggy +7126 M train-clean-360 pekein +7127 M test-clean Bill Kneeland +7128 M train-clean-360 morganreads +7131 F train-other-500 Eden Rea-Hedrick +7134 M train-clean-360 Steve Jackson +7135 F train-other-500 Maggie Smallwood +7138 F train-other-500 CaprishaPage +7139 M train-clean-360 gabrielv +7140 F train-clean-360 Deanna Bovee +7143 F train-other-500 Minni Ang +7145 F train-clean-360 Lita Ledesma +7147 M train-other-500 S.Nevets +7148 F train-clean-100 Vickie Ranz +7150 M train-other-500 asterix +7155 M train-other-500 pklipp +7169 M train-clean-360 Ryan Ransom +7170 M train-other-500 alanmapstone +7176 M test-clean KalenXI +7177 F train-other-500 shana +7178 F train-clean-100 J.K. Neely +7188 M train-clean-360 Marty +7189 M train-other-500 doonaboon +7190 M train-clean-100 Tony Posante +7197 F train-other-500 Christine Richardson +7198 F train-other-500 Daniela Austin +7199 F train-other-500 T.K. Kirven +7205 F train-other-500 genierose +7208 M train-other-500 KK +7215 M train-other-500 BensonBrunswin +7218 F train-other-500 MJ Franck +7220 F train-other-500 LynnAlison +7223 F train-other-500 eye hear voices +7226 M train-clean-100 Jonathan Moore +7228 M train-other-500 Peter John Keeble +7229 F train-clean-360 Linda Ciano +7238 M train-other-500 Uday Sagar +7239 M train-other-500 manofwealth +7240 F train-clean-360 Lucretia B. +7241 M train-clean-360 AdrianBisson +7242 F train-other-500 Renate +7245 F train-clean-360 Laura Atkinson +7246 F train-other-500 Lyn Silva +7247 M train-clean-360 Robert Hoffman +7250 F train-other-500 A. J. Elliot +7255 F train-other-500 Jendia +7258 M train-clean-360 acloward +7263 F train-other-500 rachaelg +7264 M train-clean-100 Sean McClain +7265 F train-other-500 Annie +7276 F train-clean-360 ASPotter +7277 F train-other-500 Jenny Bradshaw +7278 M train-clean-100 Jon Smith +7285 F train-clean-360 Christina M. Glavas +7286 F train-clean-360 kindfish +7294 F train-clean-360 Muriel +7297 F train-clean-360 Mudlark +7299 F train-other-500 Barbara Miller +7301 M train-other-500 jonseverity +7302 F train-clean-100 Asta1234 +7307 M train-other-500 Anthony Ogus +7312 M train-clean-100 nkneer +7313 M train-clean-360 Mark DeVol +7314 M train-clean-360 Rick Cahill +7315 F train-other-500 Charlotte Duckett +7316 F train-clean-360 Joy S Grape +7318 F train-clean-360 Anise +7320 F train-other-500 Monika Rolley +7326 M train-other-500 Adam +7327 M train-other-500 Tommy Howell +7331 F train-other-500 AnabelleC +7333 F train-other-500 Lydia Paterson +7335 F train-clean-360 Kathryn Louise +7337 F train-other-500 Adina Owen +7338 M train-other-500 Eberle Thomas +7339 M train-clean-360 wildalaska +7342 F train-clean-360 Carol +7346 M train-other-500 James Bendall +7348 F train-other-500 acousticwave +7354 M train-other-500 TrevorD777 +7357 M train-other-500 rebcult +7360 F train-other-500 Caroline Hemmerly Kunkle +7367 M train-clean-100 NIneFive83 +7376 F train-other-500 Nyssa E. Schmidt +7383 F train-clean-360 Meg Cowan +7384 F train-clean-360 Diana Dolan +7387 F train-other-500 Tara Dow +7389 M train-other-500 Steve C +7391 M train-other-500 Cary Simz +7392 F train-other-500 MicheleW +7395 F train-clean-360 Katherine +7398 F train-clean-360 Marie Daum +7402 M train-clean-100 Canby Ibarra +7408 M train-other-500 David Clarke +7416 F train-clean-360 Jeanni Hall +7423 M train-other-500 Gilles G. Le Blanc +7424 M train-other-500 Christopher Smith +7433 F train-other-500 Gabriela Cowan +7434 F train-clean-360 Emily Feuka +7436 M train-other-500 Gregory Eccles +7437 M train-clean-360 Graham McMillan +7445 F train-clean-360 Emily Anderson +7447 M train-clean-100 dasbury +7448 M train-other-500 Kevin Johnson +7460 M train-clean-360 Larry Beasley +7463 M train-other-500 Pete Milan +7467 F train-other-500 freshface +7475 M train-clean-360 Jared Hess +7478 M train-clean-360 Dan Froese +7480 F train-other-500 Tricia F. +7481 F train-clean-360 Karen Howard +7484 M train-clean-360 Tyran Rexx +7491 F train-other-500 Lilaread +7492 F train-other-500 Scheherazade +7495 F train-clean-360 Arlene Joyce +7498 F train-clean-360 MarianneVictoria +7502 M train-other-500 mikefernlea +7505 M train-clean-100 Ron Lockhart +7507 F train-other-500 UnaVersal +7510 F train-other-500 Kathrine Engan +7511 F train-clean-100 Sherri Vance +7512 M train-other-500 dudeman +7514 F train-other-500 Barbara Baker +7515 F train-clean-360 Cynthia Moyer +7517 F train-clean-100 Raz Mason +7518 M train-clean-360 TJDasch +7520 M train-clean-360 Richard Jackson +7522 F train-other-500 Kelsey P +7525 F train-clean-360 Jeannie Tirado +7538 M train-clean-360 Logan West +7540 M train-clean-360 Christopher Webber +7552 F train-other-500 KateC +7553 M train-clean-360 BDSEmpire +7555 F train-clean-360 Suebee +7556 M train-other-500 Alex Lau +7558 M train-clean-360 J. A. Cook +7559 M train-other-500 gassumpcao +7561 M train-other-500 Shawn Bayern +7565 M train-other-500 Derek +7569 M train-clean-360 Stephen L. Moss +7584 F train-other-500 kandice stehlik +7585 M train-other-500 MaxSitting +7594 F train-clean-360 Kristel Tretter +7597 F train-other-500 Inah Derby +7601 M dev-other Malone +7603 M train-other-500 Wupperhippo +7607 F train-other-500 C F de Rosset +7608 M train-other-500 rookieblue +7609 F train-other-500 detroitreads +7618 M train-other-500 Stipsky +7635 F train-clean-100 Judy Guinan +7640 M train-other-500 Dan Darbandi +7641 M dev-other Moromis +7644 F train-other-500 Raphael Platt +7647 M train-clean-360 Daniel T. Miller +7649 F train-other-500 Lanerd +7654 M train-other-500 Word And Mouth +7657 F train-clean-360 shalclark +7665 M train-clean-360 Christian Mitchell +7672 M train-other-500 Cooper Leith +7679 F train-other-500 Libby Gohn +7683 F train-other-500 Carolyne +7687 F train-other-500 Demosthenes +7688 M train-clean-360 Leslie Walden +7691 M train-other-500 engineerdst +7697 M dev-other JustinJYN +7699 F train-other-500 DylanTG +7700 M train-other-500 Dave Wills +7702 F train-other-500 Linda Jenner +7704 M train-clean-360 Robert Harder +7705 M train-clean-360 Timothy Luke +7708 F train-other-500 Savannah +7713 M train-other-500 Sacha Chander +7717 F train-clean-360 Astrid Fingerhut +7720 M train-clean-360 jeffwiltzius +7729 M test-clean Tim Bower +7730 F train-clean-360 Fiddlesticks +7732 M train-clean-360 Daniel Vimont +7733 F train-clean-360 Jewel Raquel +7737 F train-other-500 Elisabeth Sollog +7739 F train-clean-360 Melissa Burns-Price +7746 M train-other-500 Tom Causby +7749 M train-other-500 Sammy Bean +7752 F train-clean-360 Samantha Miles +7754 F train-clean-360 Iridescat +7756 F train-other-500 Lynne Thompson +7762 F train-other-500 Fannie +7764 M train-other-500 Mike Nelson +7766 F train-clean-360 Maryanka +7769 M train-other-500 David Biro +7777 M train-clean-360 Jacob Paine +7780 F train-clean-100 tazzle +7783 F train-clean-360 Rebecca Owens +7786 M train-other-500 Kevin W. Davidson +7789 F train-clean-360 Lauren Jane +7794 F train-clean-100 mlcui +7795 M train-other-500 Cubrick +7796 M train-other-500 Raybrite +7800 F train-clean-100 Arie +7802 F train-clean-360 Samantha J Gubitz +7809 M train-clean-360 D. T. McGregor +7816 F train-clean-360 Stefanie Heinrichs +7823 M train-other-500 Bart in 't Veld +7825 M train-clean-360 PDyer +7826 M train-other-500 Dave Shaw +7828 M train-clean-360 Dan Mason +7832 M train-clean-360 DJRickyV +7833 M train-clean-360 Jesse Crisp-Sears +7835 M train-other-500 kageokami +7837 M train-clean-360 Alfian +7839 F train-other-500 Loveday +7843 F train-other-500 Annette McGuinness +7848 M train-other-500 bala +7850 F dev-clean Jill Engle +7859 F train-clean-100 xinamarieuhl +7867 M train-clean-360 nomorejeffs +7868 F train-clean-360 Chessie Joy +7871 M train-other-500 Rocit +7874 M train-clean-360 dgulino +7879 F train-other-500 deongines +7881 M train-clean-360 Sam Naishtat +7883 F train-other-500 Annapurna +7886 F train-other-500 Mariah Lyons +7892 F train-other-500 Alexis Castro +7898 F train-other-500 Coreena +7902 M test-other Kyle Van DeGlast +7909 M train-clean-360 MGreenberg +7910 F train-clean-360 southernemma +7912 M train-other-500 Nathan Dickey +7923 M train-other-500 amaskill +7925 F train-other-500 Etel Buss +7926 M train-clean-360 Steven Reynolds +7932 F train-clean-360 Tammy Stalcup +7933 F train-clean-360 Highlandoaks +7938 M train-clean-360 Goss45 +7939 M train-clean-360 DPranitis +7942 M train-other-500 Luke Sartor +7945 F train-clean-360 Lois Browne +7946 F train-other-500 Flash +7949 M train-clean-360 Jon Miller +7956 M train-clean-360 Devon Purtz +7957 M train-clean-360 Wiley Combs +7959 F train-clean-360 Lily Took +7962 F train-clean-360 Jolie O'Dell +7967 F train-clean-360 Lois Hill +7975 M test-other William A Crenshaw +7976 M dev-clean JenniferRutters +7981 M train-clean-360 timothyFR +7982 F train-clean-360 Lori Arsenault +7988 M train-other-500 JaySands +7991 M train-clean-360 Aaron Weber +7994 F train-clean-360 Marian Cervassi +7995 F train-clean-360 Marie Hoffman +7997 M train-other-500 Tom Merritt +8005 F train-other-500 Julia Wells +8006 M train-clean-360 TyroneS +8008 M train-clean-360 Paul Adamson +8009 F train-other-500 Frances Brown +8011 M train-clean-360 Greg Giordano +8012 F train-other-500 WoollyBee +8014 F train-clean-100 constatine +8015 F train-other-500 Underhill +8023 M train-other-500 JamesMcAndrew +8028 M train-clean-360 Tom Geller +8033 M train-other-500 geoffbl +8040 F train-other-500 NatalieOram +8042 M train-other-500 Jack Watson Warr +8044 F train-other-500 Sarah Hannah +8050 M train-clean-360 Jake Woldstad +8051 F train-clean-100 Maria Kasper +8057 F train-clean-360 Linda Dougherty +8058 M train-other-500 RLC +8063 M train-clean-100 Robert Snoza +8066 M train-clean-360 Tim Cote +8071 F train-other-500 Vanessa Garcia +8072 F train-other-500 Kimberly Krause +8075 F train-clean-360 Melora +8080 F train-clean-360 e.a.zokaites +8087 M train-other-500 Arnie Horton +8088 M train-clean-100 Jason Bolestridge +8095 M train-clean-100 Theodulf +8097 F train-clean-360 lewildesen +8098 M train-clean-100 Arnold +8108 M train-clean-100 drakaunus +8112 F train-other-500 Christine Lamberton +8113 F train-clean-360 PennyAnn +8118 F train-clean-360 Katie McClain +8119 M train-clean-360 Malcolm Cameron +8123 F train-clean-100 Sheila Wood +8131 M test-other Christian Alexander +8138 M train-clean-360 Lee Smalley +8142 M train-clean-360 Timothy Lucas +8143 M train-other-500 Nick Whitley +8148 F train-other-500 Xiph +8152 M train-clean-360 Bigcowfeet +8156 F train-other-500 Brooke Cunningham +8163 F train-clean-360 Greengecko +8164 M train-other-500 Rob Board +8168 F train-other-500 Kiera Davidson +8169 M train-other-500 tovarisch +8172 M train-other-500 Alan Weyman +8173 F dev-other emmablob +8176 M train-clean-360 Jon Kerfoot +8180 F train-other-500 Elanor Sakamoto +8183 F train-clean-360 Cheri Jordan +8188 M test-other Matthew Calvin +8190 F train-clean-360 Lisa Phelps Gonzalez +8193 F train-clean-360 helengraves +8194 F train-clean-360 MsMO +8195 M train-clean-360 Mr Krause +8197 F train-other-500 Victoria P +8199 F train-other-500 maryagneskatherine +8200 M train-other-500 Dan S +8208 M train-other-500 zaanta +8215 M train-other-500 readread +8222 M train-clean-360 Greg Golding +8224 M test-clean Leanne Kinkopf +8225 M train-clean-360 Matt Lusher +8226 M train-clean-100 Adam Picot +8228 F train-clean-360 hgal2010 +8230 M test-clean David Jenkins +8238 F train-clean-100 Madam Fickle +8240 F train-other-500 Gertrude Durette +8242 F train-other-500 Anita Slusser +8245 M train-other-500 gscheids +8246 M train-other-500 beeveeo +8250 M train-other-500 Stephen Gibbons +8254 F dev-other Jeana Wei +8259 F train-other-500 Lawrence +8262 M train-other-500 Jack Powell +8266 M train-clean-360 Jeff K. +8272 F train-other-500 Haili +8273 M train-other-500 Lucas Boulding +8280 F test-other AlaynaMay +8288 M dev-other Wayne Donovan +8291 M train-other-500 Elliot Gage +8295 F train-other-500 Susan Morin +8296 F train-other-500 Bria Snow +8297 M dev-clean David Mecionis +8300 F train-clean-360 Judith Parker +8302 F train-other-500 Charlotte Day +8307 M train-other-500 Nick Bulka +8312 F train-clean-100 Jaimie Noy +8316 M train-other-500 Rocket Rodger +8321 F train-other-500 SamR +8322 M train-other-500 yeknod +8324 F train-clean-100 Kathy Wright +8328 M train-other-500 William Gavula +8329 F train-clean-360 Betty Perry +8334 M train-other-500 Doug Reed +8337 F train-other-500 Claudia Salto +8346 M train-other-500 Al Rocca +8347 M train-clean-360 Terry Torres +8356 M train-other-500 Paul Mazumdar +8367 M train-other-500 ophiuroidea +8382 F train-other-500 Claire Schreuder +8388 F train-clean-360 Jacki Horn +8389 M train-other-500 Steven Bateman +8392 F train-other-500 Katharina Huang +8394 M train-other-500 Matthew Walker +8396 M train-clean-360 gloriousjob +8401 M train-clean-360 Alexander Hatton +8404 F train-clean-360 Lynne Ray +8410 F train-clean-360 Shauna Kennett +8413 M train-other-500 PaulMichael1084 +8414 F train-other-500 jtueller +8415 F train-other-500 Rusty Dancer +8419 M train-clean-100 Jon Kissack +8421 F train-clean-360 Jackie Drown +8422 M train-other-500 pjhoury +8424 F train-other-500 Schums +8425 M train-clean-100 Larry Wilson +8430 F train-other-500 shihping +8432 F train-other-500 Emma Joyce +8441 F train-other-500 ryoko +8443 F train-other-500 Marsha Payne +8444 M train-other-500 Anthony Webster +8445 M train-other-500 Brett G. Hirsch +8447 F train-other-500 Anastasiia Solokha +8455 M test-clean thecheops +8459 M train-clean-360 sdaeley17 +8461 F test-other Allie Cingi +8463 F test-clean Michele Fry +8464 M train-clean-360 HappyMiamiDad +8465 F train-clean-100 TinaNygard2 +8466 M train-other-500 Chris Cartwright +8468 F train-clean-100 Jennifer Dorr +8470 F train-other-500 Rebecca Braunert-Plunkett +8474 M train-clean-360 Scott Snowman +8476 F train-other-500 Julie Mansius +8479 M train-clean-360 R.W. Rushing +8490 M train-clean-360 Matt Parker +8494 M train-clean-360 Bill Yallalee +8498 M train-clean-360 Ian Quinlan +8499 M train-other-500 ReadAllDay +8500 M train-other-500 Ravi Shankar +8506 F train-clean-360 Denise Nordell +8527 M train-clean-360 Gary Bohannon +8531 M train-other-500 imenadel +8534 M train-clean-360 Chris Clark +8536 M train-other-500 Jonathan Brubaker +8543 F train-other-500 luckyemma +8544 F train-other-500 Pooja DSr +8545 F train-clean-360 Joanne Rochon +8555 F test-clean Michelle Goode +8565 M train-other-500 Patrick Eaton +8573 F train-clean-360 Paige G +8575 M train-clean-360 Jeremy Robertson +8576 M train-other-500 Phil Schempf +8580 M train-clean-100 Gary Dana +8587 M train-other-500 Drew Johnson +8590 M train-other-500 bobrose +8591 F train-clean-360 Jude Somers +8592 M train-clean-360 Dylan Posa +8605 F train-clean-360 Kate Sterner +8609 M train-clean-100 noblesavage +8619 M train-clean-360 Tad E. +8625 F train-other-500 Ellen Preckel +8629 M train-clean-100 Shivansh Dhar +8630 M train-clean-100 Eduardo +8631 M train-other-500 Edward Kirkby +8632 M train-other-500 Geremia +8635 M train-clean-360 ACBowgus +8643 M train-clean-360 jabc1950 +8644 F train-other-500 Eenae +8664 F train-other-500 Pam Castille +8666 M train-other-500 josembi +8671 M train-other-500 Simon Smoke +8675 M train-other-500 Larry Greene +8677 F train-clean-360 KHand +8678 F train-other-500 Amy +8684 F train-clean-360 Alison Stewart +8687 M train-clean-360 EccentricOwl +8699 F train-clean-360 Jessica Atha +8705 M train-clean-360 dsilber01 +8710 M train-other-500 Ralph Crown +8713 M train-clean-360 Expatriate +8718 F train-clean-360 Deena Rhoads +8722 F train-clean-360 Emily Maynard +8725 M train-clean-360 Gary Ericson +8742 M train-clean-360 Adam Taylor +8747 M train-clean-100 DeanOBuchanan +8753 M train-other-500 Walt Allan +8758 M train-clean-360 Krzysztof Rowinski +8765 M train-other-500 jciesielski +8770 M train-clean-100 Paul Simonin +8771 F train-clean-360 Anna Millard +8772 M train-clean-360 Martin Reyto +8776 F train-clean-360 Taliesin +8778 F train-other-500 Francoise +8786 M train-clean-360 Bruce Kachuk +8791 M train-clean-360 ScottReyonoldsVoice +8797 M train-clean-100 Sean Grabosky +8799 M train-other-500 Peter Tucker +8803 M train-other-500 Ben Lindsey-Clark +8808 M train-other-500 davidpr +8820 M train-clean-360 Ignare +8824 M train-clean-360 Mark Johnston +8825 F train-clean-360 Erin Schellhase +8838 M train-clean-100 Kevin Owens +8842 F dev-clean Mary J +8846 F train-other-500 MariaS +8848 M train-clean-360 Craig Kenneth Bryant +8855 M train-clean-360 Eric Metzler +8867 F train-other-500 sorbet87 +8875 M train-clean-360 David D'Huet +8879 M train-clean-360 Son of the Exiles +8887 M train-clean-360 Andrew Hernandez +8897 F train-other-500 beyondutopia +8975 F train-clean-100 Daisy Flaim +9000 M train-other-500 Ramon Escamilla +9022 F train-clean-360 Claire M +9023 F train-clean-360 P. J. Morgan +9026 F train-clean-360 Tammy Porter diff --git a/config/vocab.txt b/config/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..f43629fcd41099b67efcaeed6470c66b6417d598 --- /dev/null +++ b/config/vocab.txt @@ -0,0 +1,30551 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[SILENCE] +[FULL STOP] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ +[COMMA] +[EXCLAMATION MARK] +[QUESTION MARK] +[SEMICOLON] +aɪ +aʊ +dʒ +eɪ +oʊ +tʃ +ɔɪ +ɜ˞ +ˌaɪ +ˌaʊ +ˌeɪ +ˌi +ˌoʊ +ˌu +ˌæ +ˌɑ +ˌɔ +ˌɔɪ +ˌɛ +ˌɜ˞ +ˌɪ +ˌʊ +ˌʌ diff --git a/config/vocab_phonemes.txt b/config/vocab_phonemes.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f1a71a0041aa66b5cf3e4c4288276637a58c57a --- /dev/null +++ b/config/vocab_phonemes.txt @@ -0,0 +1,63 @@ +[MASK] +[PAD] +[UNK] +[COMMA] +[EXCLAMATION MARK] +[FULL STOP] +[QUESTION MARK] +[SEMICOLON] +[SILENCE] +aɪ +aʊ +b +d +dʒ +eɪ +f +h +i +j +k +l +m +n +oʊ +p +s +t +tʃ +u +v +w +z +æ +ð +ŋ +ɑ +ɔ +ɔɪ +ɛ +ɜ˞ +ɡ +ɪ +ɹ +ʃ +ʊ +ʌ +ʒ +ˌaɪ +ˌaʊ +ˌeɪ +ˌi +ˌoʊ +ˌu +ˌæ +ˌɑ +ˌɔ +ˌɔɪ +ˌɛ +ˌɜ˞ +ˌɪ +ˌʊ +ˌʌ +θ diff --git a/demo/__init__.py b/demo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/demo/config.py b/demo/config.py new file mode 100644 index 0000000000000000000000000000000000000000..ce58a18e96ab9ddd17ce9147678c86fad436fda5 --- /dev/null +++ b/demo/config.py @@ -0,0 +1,109 @@ +speakers_delightful_22050 = { + "Dan Threetrees": 50, + "Cori Samuel": 52, + "Linda Wilcox": 63, + "Christiane Levesque": 64, + "Fox in the Stars": 80, + "Chris Goringe": 91, + "Maureen S. O'Brien": 102, + "Kevin McAsh": 108, + "Brenda Dayne": 118, + "Sean McGaughey": 123, + "Ed Good": 164, + "Paul Harvey": 171, + "Nocturna": 192, + "Joy Chan": 218, + "fieldsofgold": 248, + "Eric Connover": 308, + "Carl Vonnoh, III": 314, + "Brooks Seveer": 373, + "swroot": 394, + "Christabel": 414, + "Kiki Baessell": 417, + "RedToby": 425, + "Heather Duncan": 450, + "Christian Pecaut": 469, + "Scott Sherris": 481, + "Madame Tusk": 517, + "Miranda Stinson": 544, + "Daniel Shorten": 574, + "Jo": 596, + "Anne-Marie": 604, + "PJ": 617, + "Cynthia Zocca": 644, + "Micah": 645, + "Scarlett!": 649, + "cucciasv": 653, + "James Gladwin": 661, + "Kelly Dougherty": 666, + "Jan Baxter": 668, + "Nikki Sullivan": 685, + "Sheila Morton": 710, + "Tysto": 732, + "Pete Williams, Pittsburgh, PA": 770, + "Alec Daitsman": 782, + "David Kleparek": 791, + "FirstKnight": 804, + "texttalker": 817, + "kristiface": 822, + "Rachel Lintern": 835, + "Jennifer Lott": 851, + "mpetranech": 881, + "Quentin Manuel": 908, + "Jane Greensmith": 923, + "Petra": 929, + "Raerity": 932, + "Parrot": 995, + "mjbrichant": 1041, + "Martin Geeson": 1109, + "Linda Andrus": 1135, + "Jonathan Burchard": 1142, + "Dale A. Bade": 1204, + "Troy Bond": 1230, + "Sarah LuAnn": 1247, + "garbageman99": 1281, + "davidb": 1302, + "Savanna Herrold": 1370, + "Angel5": 1402, + "Preston Scrape": 1414, + "browneyedgirl32382": 1433, + "P Moscato": 1440, + "Joyce Couch": 1596, + "Sharon Omi": 1656, + "Steve Belleguelle": 1712, + "Caroline Driggs": 1718, + "Vinnie Tesla": 1728, + "anjieliu": 1741, + "Yvonne Smith": 1755, + "Sarah Crampton": 1783, + "Vince Dee": 1805, + "Rebecca King": 1808, + "Kendall Ashyby": 1836, + "NastassiaS": 1869, + "acloward": 2056, + "Eberle Thomas": 2084, + "Larry Beasley": 2114, + "Pete Milan": 2115, + "Suebee": 2143, + "Sammy Bean": 2192, + "Mike Nelson": 2197, + "Samantha J Gubitz": 2209, + "Haili": 2341, + "Alexander Hatton": 2371, + "KHand": 2439, + "Deena Rhoads": 2447, + "Erin Schellhase": 2468, +} + +speakers_hifi_tts = { + "Cori Samuel": 92, + "Phil Benson": 6097, + "John Van Stan": 9017, + "Mike Pelton": 6670, + "Tony Oliva": 6671, + "Maria Kasper": 8051, + "Helen Taylor": 9136, + "Sylviamb": 11614, + "Celine Major": 11697, + "LikeManyWaters": 12787, +} diff --git a/en_us_cmudict_ipa_forward.pt b/en_us_cmudict_ipa_forward.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5e6d969a6ec047297aa08875bab7520b149c3ff --- /dev/null +++ b/en_us_cmudict_ipa_forward.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cadce3d77597b55e772799cb46994ab29a460f1a62a87207b52f3cdb29894e02 +size 65637046 diff --git a/epoch=5816-step=390418.ckpt b/epoch=5816-step=390418.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c4e2e1343ff5742fcc6047e2c2261ce2a0607bbc --- /dev/null +++ b/epoch=5816-step=390418.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65f41d587e37cc12a05e43c9f0ed107dcca634f3987bee7facec6f62571aeda9 +size 2381456584 diff --git a/mocks/audio_example.wav b/mocks/audio_example.wav new file mode 100644 index 0000000000000000000000000000000000000000..004a33532ea2547c10c0074b967733ba91edd9f8 Binary files /dev/null and b/mocks/audio_example.wav differ diff --git a/mocks/data/Alice/0001.pt b/mocks/data/Alice/0001.pt new file mode 100644 index 0000000000000000000000000000000000000000..99a25b9234a35137d7875db1f642fc1677631662 --- /dev/null +++ b/mocks/data/Alice/0001.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f5942c46d80a7517d7979589b5c2f6613138fedb921b23ed0cdd7b065f2409 +size 97172 diff --git a/mocks/data/Bob/0002.pt b/mocks/data/Bob/0002.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ab1537802ba1ba577bb43296c74bf738ca9faa0 --- /dev/null +++ b/mocks/data/Bob/0002.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2765d71f271f668d80c266deb449cee5d94ad19e4949db10f4b50cf53b6d1c5e +size 97172 diff --git a/mocks/data/Charlie/0003.pt b/mocks/data/Charlie/0003.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c48a79ccffa0dab8698d2aeba43dee8f22e60fb --- /dev/null +++ b/mocks/data/Charlie/0003.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f3de9a2b066779141176513f2e0ee80d0ec87157fd30e0c59289562d58f9fe2 +size 97172 diff --git a/mocks/metadata.txt b/mocks/metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..256b9d67e990f5bf26943693bdf3ccadd70a25c9 --- /dev/null +++ b/mocks/metadata.txt @@ -0,0 +1,3 @@ +0001|Alice +0002|Bob +0003|Charlie \ No newline at end of file diff --git a/mocks/speakers.json b/mocks/speakers.json new file mode 100644 index 0000000000000000000000000000000000000000..95934ceb63044831dc2ca8d883a7f3e526b83476 --- /dev/null +++ b/mocks/speakers.json @@ -0,0 +1,5 @@ +{ + "Alice": 0, + "Bob": 1, + "Charlie": 2 +} \ No newline at end of file diff --git a/mocks/test_compute_yin.npy b/mocks/test_compute_yin.npy new file mode 100644 index 0000000000000000000000000000000000000000..47049cd37181f8cbc8d4e6431847572321a9fce1 --- /dev/null +++ b/mocks/test_compute_yin.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9a4db041f1f2f053d1477775dc177d27989d5186c0849cc2ab24e337c2c4115 +size 5632 diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35c96568d6e28f38ad63c720464521171d18f154 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1 @@ +from .delightful_hifi import DelightfulHiFi diff --git a/models/config/__init__.py b/models/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..602498f13e338ca48f3a8e14c4f2f5155f41f2c2 --- /dev/null +++ b/models/config/__init__.py @@ -0,0 +1,3 @@ +from .configs import * +from .langs import * +from .symbols import * diff --git a/models/config/configs.py b/models/config/configs.py new file mode 100644 index 0000000000000000000000000000000000000000..b9583ca7a6827886e964d3ad66c934ba7369cc2d --- /dev/null +++ b/models/config/configs.py @@ -0,0 +1,438 @@ +from dataclasses import dataclass, field +from typing import List, Literal, Tuple, Union + +PreprocessLangType = Literal["english_only", "multilingual"] + + +@dataclass +class STFTConfig: + filter_length: int + hop_length: int + win_length: int + n_mel_channels: int + mel_fmin: int + mel_fmax: int + + +# Base class used with the Univnet vocoder +@dataclass +class PreprocessingConfig: + language: PreprocessLangType + stft: STFTConfig + sampling_rate: int = 22050 + min_seconds: float = 0.5 + max_seconds: float = 6.0 + use_audio_normalization: bool = True + workers: int = 8 + + +@dataclass +class PreprocessingConfigUnivNet(PreprocessingConfig): + stft: STFTConfig = field( + default_factory=lambda: STFTConfig( + filter_length=1024, + hop_length=256, + win_length=1024, + n_mel_channels=100, # univnet + mel_fmin=20, + mel_fmax=11025, + ), + ) + + +@dataclass +class PreprocessingConfigHifiGAN(PreprocessingConfig): + stft: STFTConfig = field( + default_factory=lambda: STFTConfig( + filter_length=1024, + hop_length=256, + win_length=1024, + n_mel_channels=80, # For univnet 100 + mel_fmin=20, + mel_fmax=11025, + ), + ) + + def __post_init__(self): + r"""It modifies the 'stft' attribute based on the 'sampling_rate' attribute. + If 'sampling_rate' is 44100, 'stft' is set with specific values for this rate. + If 'sampling_rate' is not 22050 or 44100, a ValueError is raised. + + Raises: + ValueError: If 'sampling_rate' is not 22050 or 44100. + """ + if self.sampling_rate == 44100: + self.stft = STFTConfig( + filter_length=2048, + hop_length=512, # NOTE: 441 ?? https://github.com/jik876/hifi-gan/issues/116#issuecomment-1436999858 + win_length=2048, + n_mel_channels=80, # Based on https://github.com/jik876/hifi-gan/issues/116 + mel_fmin=20, + mel_fmax=11025, + ) + if self.sampling_rate not in [22050, 44100]: + raise ValueError("Sampling rate must be 22050 or 44100") + + +@dataclass +class AcousticTrainingOptimizerConfig: + learning_rate: float + weight_decay: float + lr_decay: float + betas: Tuple[float, float] = (0.9, 0.98) + eps: float = 0.000000001 + grad_clip_thresh: float = 1.0 + warm_up_step: float = 4000 + anneal_steps: List[int] = field(default_factory=list) + anneal_rate: float = 0.3 + + +@dataclass +class AcousticFinetuningConfig: + batch_size = 5 + grad_acc_step = 3 + train_steps = 30000 + log_step = 100 + synth_step = 250 + val_step = 4000 + save_step = 250 + freeze_bert_until = 0 + mcd_gen_max_samples = 400 + only_train_speaker_until = 5000 + optimizer_config: AcousticTrainingOptimizerConfig = field( + default_factory=lambda: AcousticTrainingOptimizerConfig( + learning_rate=0.0002, + weight_decay=0.001, + lr_decay=0.99999, + ), + ) + + +@dataclass +class AcousticPretrainingConfig: + batch_size = 5 + grad_acc_step = 5 + train_steps = 500000 + log_step = 20 + synth_step = 250 + val_step = 4000 + save_step = 1000 + freeze_bert_until = 4000 + mcd_gen_max_samples = 400 + only_train_speaker_until = 0 + optimizer_config: AcousticTrainingOptimizerConfig = field( + default_factory=lambda: AcousticTrainingOptimizerConfig( + learning_rate=0.0002, + weight_decay=0.01, + lr_decay=1.0, + ), + ) + + +AcousticTrainingConfig = Union[AcousticFinetuningConfig, AcousticPretrainingConfig] + + +@dataclass +class ConformerConfig: + n_layers: int + n_heads: int + n_hidden: int + p_dropout: float + kernel_size_conv_mod: int + kernel_size_depthwise: int + with_ff: bool + + +@dataclass +class ReferenceEncoderConfig: + bottleneck_size_p: int + bottleneck_size_u: int + ref_enc_filters: List[int] + ref_enc_size: int + ref_enc_strides: List[int] + ref_enc_pad: List[int] + ref_enc_gru_size: int + ref_attention_dropout: float + token_num: int + predictor_kernel_size: int + + +@dataclass +class VarianceAdaptorConfig: + n_hidden: int + kernel_size: int + emb_kernel_size: int + p_dropout: float + n_bins: int + + +@dataclass +class AcousticLossConfig: + ssim_loss_alpha: float + mel_loss_alpha: float + aligner_loss_alpha: float + pitch_loss_alpha: float + energy_loss_alpha: float + u_prosody_loss_alpha: float + p_prosody_loss_alpha: float + dur_loss_alpha: float + binary_align_loss_alpha: float + binary_loss_warmup_epochs: int + + +@dataclass +class AcousticENModelConfig: + speaker_embed_dim: int = 1024 + lang_embed_dim: int = 1 + encoder: ConformerConfig = field( + default_factory=lambda: ConformerConfig( + n_layers=6, + n_heads=8, + n_hidden=512, + p_dropout=0.1, + kernel_size_conv_mod=7, + kernel_size_depthwise=7, + with_ff=True, + ), + ) + decoder: ConformerConfig = field( + default_factory=lambda: ConformerConfig( + n_layers=6, + n_heads=8, + n_hidden=512, + p_dropout=0.1, + kernel_size_conv_mod=11, + kernel_size_depthwise=11, + with_ff=True, + ), + ) + reference_encoder: ReferenceEncoderConfig = field( + default_factory=lambda: ReferenceEncoderConfig( + bottleneck_size_p=4, + bottleneck_size_u=256, + ref_enc_filters=[32, 32, 64, 64, 128, 128], + ref_enc_size=3, + ref_enc_strides=[1, 2, 1, 2, 1], + ref_enc_pad=[1, 1], + ref_enc_gru_size=32, + ref_attention_dropout=0.2, + token_num=32, + predictor_kernel_size=5, + ), + ) + variance_adaptor: VarianceAdaptorConfig = field( + default_factory=lambda: VarianceAdaptorConfig( + n_hidden=512, + kernel_size=5, + emb_kernel_size=3, + p_dropout=0.5, + n_bins=256, + ), + ) + loss: AcousticLossConfig = field( + default_factory=lambda: AcousticLossConfig( + ssim_loss_alpha=1.0, + mel_loss_alpha=1.0, + aligner_loss_alpha=1.0, + pitch_loss_alpha=1.0, + energy_loss_alpha=1.0, + u_prosody_loss_alpha=0.25, + p_prosody_loss_alpha=0.25, + dur_loss_alpha=1.0, + binary_align_loss_alpha=0.1, + binary_loss_warmup_epochs=10, + ), + ) + + +@dataclass +class AcousticMultilingualModelConfig: + speaker_embed_dim: int = 1024 + lang_embed_dim: int = 256 + encoder: ConformerConfig = field( + default_factory=lambda: ConformerConfig( + n_layers=6, + n_heads=8, + n_hidden=512, + p_dropout=0.1, + kernel_size_conv_mod=7, + kernel_size_depthwise=7, + with_ff=True, + ), + ) + decoder: ConformerConfig = field( + default_factory=lambda: ConformerConfig( + n_layers=6, + n_heads=8, + n_hidden=512, + p_dropout=0.1, + kernel_size_conv_mod=11, + kernel_size_depthwise=11, + with_ff=True, + ), + ) + reference_encoder: ReferenceEncoderConfig = field( + default_factory=lambda: ReferenceEncoderConfig( + bottleneck_size_p=4, + bottleneck_size_u=256, + ref_enc_filters=[32, 32, 64, 64, 128, 128], + ref_enc_size=3, + ref_enc_strides=[1, 2, 1, 2, 1], + ref_enc_pad=[1, 1], + ref_enc_gru_size=32, + ref_attention_dropout=0.2, + token_num=32, + predictor_kernel_size=5, + ), + ) + variance_adaptor: VarianceAdaptorConfig = field( + default_factory=lambda: VarianceAdaptorConfig( + n_hidden=512, + kernel_size=5, + emb_kernel_size=3, + p_dropout=0.5, + n_bins=256, + ), + ) + loss: AcousticLossConfig = field( + default_factory=lambda: AcousticLossConfig( + ssim_loss_alpha=1.0, + mel_loss_alpha=1.0, + aligner_loss_alpha=1.0, + pitch_loss_alpha=1.0, + energy_loss_alpha=1.0, + u_prosody_loss_alpha=0.25, + p_prosody_loss_alpha=0.25, + dur_loss_alpha=1.0, + binary_align_loss_alpha=0.1, + binary_loss_warmup_epochs=10, + ), + ) + + +AcousticModelConfigType = Union[AcousticENModelConfig, AcousticMultilingualModelConfig] + + +@dataclass +class VocoderBasicConfig: + segment_size: int = 16384 + learning_rate: float = 0.0001 + adam_b1: float = 0.5 + adam_b2: float = 0.9 + lr_decay: float = 0.995 + synth_interval: int = 250 + checkpoint_interval: int = 250 + stft_lamb: float = 2.5 + + +@dataclass +class VocoderPretrainingConfig(VocoderBasicConfig): + batch_size: int = 14 + grad_accum_steps: int = 1 + train_steps: int = 1000000 + stdout_interval: int = 25 + validation_interval: int = 2000 + + +@dataclass +class VocoderFinetuningConfig(VocoderBasicConfig): + batch_size: int = 5 + grad_accum_steps: int = 3 + train_steps: int = 10000 + stdout_interval: int = 100 + validation_interval: int = 4000 + + +VoicoderTrainingConfig = Union[VocoderPretrainingConfig, VocoderFinetuningConfig] + + +@dataclass +class VocoderGeneratorConfig: + noise_dim: int + channel_size: int + dilations: List[int] + strides: List[int] + lReLU_slope: float + kpnet_conv_size: int + + +@dataclass +class VocoderMPDConfig: + periods: List[int] + kernel_size: int + stride: int + use_spectral_norm: bool + lReLU_slope: float + + +@dataclass +class VocoderMRDConfig: + resolutions: List[Tuple[int, int, int]] + use_spectral_norm: bool + lReLU_slope: float + + +@dataclass +class VocoderModelConfig: + gen: VocoderGeneratorConfig = field( + default_factory=lambda: VocoderGeneratorConfig( + noise_dim=64, + channel_size=32, + dilations=[1, 3, 9, 27], + strides=[8, 8, 4], + lReLU_slope=0.2, + kpnet_conv_size=3, + ), + ) + mpd: VocoderMPDConfig = field( + default_factory=lambda: VocoderMPDConfig( + periods=[2, 3, 5, 7, 11], + kernel_size=5, + stride=3, + use_spectral_norm=False, + lReLU_slope=0.2, + ), + ) + mrd: VocoderMRDConfig = field( + default_factory=lambda: VocoderMRDConfig( + resolutions=[(1024, 120, 600), (2048, 240, 1200), (512, 50, 240)], + use_spectral_norm=False, + lReLU_slope=0.2, + ), + ) + + +##################### +# HI-FI GAN CONFIGS # +##################### + + +@dataclass +class HifiGanPretrainingConfig(VocoderBasicConfig): + segment_size: int = 16384 + learning_rate: float = 0.0002 + adam_b1: float = 0.8 + adam_b2: float = 0.99 + lr_decay: float = 0.9995 + lReLU_slope: float = 0.1 + l1_factor: int = 45 + sampling_rate_acoustic: int = 22050 + sampling_rate_vocoder: int = 44100 + + +@dataclass +class HifiGanConfig: + resblock: str = "1" + upsample_rates: List[int] = field( + default_factory=lambda: [8, 8, 4, 2], + ) + upsample_kernel_sizes: List[int] = field( + default_factory=lambda: [16, 16, 4, 4], + ) + upsample_initial_channel: int = 512 + resblock_kernel_sizes: List[int] = field( + default_factory=lambda: [3, 7, 11], + ) + resblock_dilation_sizes: List[List[int]] = field( + default_factory=lambda: [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + ) diff --git a/models/config/experimental_configs.py b/models/config/experimental_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..8ba15487bdb08f5ceed10bc4c7bb0b893a7707df --- /dev/null +++ b/models/config/experimental_configs.py @@ -0,0 +1,100 @@ +from dataclasses import dataclass + + +# TODO: DEPRECATED! +@dataclass +class PostNetConfig: + p_dropout: float + postnet_embedding_dim: int + postnet_kernel_size: int + postnet_n_convolutions: int + +postnet_expetimental = PostNetConfig( + p_dropout=0.1, + postnet_embedding_dim=512, + postnet_kernel_size=5, + postnet_n_convolutions=3, +) + +# TODO: DEPRECATED! +@dataclass +class DiffusionConfig: + # model parameters + model: str + n_mel_channels: int + multi_speaker: bool + # denoiser parameters + residual_channels: int + residual_layers: int + denoiser_dropout: float + noise_schedule_naive: str + timesteps: int + shallow_timesteps: int + min_beta: float + max_beta: float + s: float + pe_scale: int + keep_bins: int + # trainsformer params + encoder_hidden: int + decoder_hidden: int + speaker_embed_dim: int + # loss params + noise_loss: str + + +diff_en = DiffusionConfig( + # model parameters + model="shallow", + n_mel_channels=100, + multi_speaker=True, + # denoiser parameters + # residual_channels=256, + # residual_channels=384, + residual_channels=100, + residual_layers=20, + denoiser_dropout=0.2, + noise_schedule_naive="vpsde", + timesteps=10, + shallow_timesteps=1, + min_beta=0.1, + max_beta=40, + s=0.008, + keep_bins=80, + pe_scale=1000, + # trainsformer params + # encoder_hidden=100, + encoder_hidden=512, + decoder_hidden=512, + # Speaker_emb + lang_emb + speaker_embed_dim=1025, + # loss params + noise_loss="l1", +) + +diff_multi = DiffusionConfig( + # model parameters + model="shallow", + n_mel_channels=100, + multi_speaker=True, + # denoiser parameters + # residual_channels=256, + residual_channels=100, + residual_layers=20, + denoiser_dropout=0.2, + noise_schedule_naive="vpsde", + timesteps=10, + shallow_timesteps=1, + min_beta=0.1, + max_beta=40, + s=0.008, + pe_scale=1000, + keep_bins=80, + # trainsformer params + encoder_hidden=512, + decoder_hidden=512, + # Speaker_emb + lang_emb + speaker_embed_dim=1280, + # loss params + noise_loss="l1", +) diff --git a/models/config/langs.py b/models/config/langs.py new file mode 100644 index 0000000000000000000000000000000000000000..f943ac048f2fe9e190037fdcdfe9f6dbb3178c3f --- /dev/null +++ b/models/config/langs.py @@ -0,0 +1,65 @@ +from dataclasses import dataclass +from typing import Dict + +from models.config import PreprocessLangType + +# TODO: now we only support english, but we need to support other languages! +SUPPORTED_LANGUAGES = [ + "bg", + "cs", + "de", + "en", + "es", + "fr", + "ha", + "hr", + "ko", + "pl", + "pt", + "ru", + "sv", + "sw", + "th", + "tr", + "uk", + "vi", + "zh", +] + +# Mappings from symbol to numeric ID and vice versa: +lang2id = {s: i for i, s in enumerate(SUPPORTED_LANGUAGES)} +id2lang = dict(enumerate(SUPPORTED_LANGUAGES)) + +@dataclass +class LangItem: + r"""A class for storing language information.""" + + phonemizer: str + phonemizer_espeak: str + nemo: str + processing_lang_type: PreprocessLangType + +langs_map: Dict[str, LangItem] = { + "en": LangItem( + phonemizer="en_us", + phonemizer_espeak="en-us", + nemo="en", + processing_lang_type="english_only", + ), +} + +def get_lang_map(lang: str) -> LangItem: + r"""Returns a LangItem object for the given language. + + Args: + lang (str): The language to get the LangItem for. + + Raises: + ValueError: If the language is not supported. + + Returns: + LangItem: The LangItem object for the given language. + """ + if lang not in langs_map: + raise ValueError(f"Language {lang} is not supported!") + return langs_map[lang] diff --git a/models/config/speakers.py b/models/config/speakers.py new file mode 100644 index 0000000000000000000000000000000000000000..0c29f6ff3257b2cf58ab1f654c7a26501057cedc --- /dev/null +++ b/models/config/speakers.py @@ -0,0 +1,451 @@ +import json +from typing import Dict, List + +# Load the ID mapping +with open("training/datasets/speaker_id_mapping_libri.json") as f: + id_mapping = json.load(f) + +# Create a reverse mapping +reverse_mapping: Dict[int, int] = {int(v): int(k) for k, v in id_mapping.items()} + +# Selected for the fine-tuning +# train-960 subset of LibriTTS +selected_speakers = [ + 574, # Daniel Shorten M train-clean-100 + 242, # J. Hall M train-other-500 + 536, # Robert Flach M train-other-500 + 82, # Andy Minter M train-other-500 + 672, # Stuart Bell M train-other-500 + 315, # Jean Crevier M train-other-500 + 628, # Bryan Ness M train-clean-100 + 61, # John Greenman M train-other-500 + 649, # Scarlett! F train-clean-360 + 105, # Marian Brown F train-clean-360 + 399, # entada F train-clean-360 + 89, # Paula Berinstein F train-clean-360 + 502, # Lee Elliott F train-other-500 + 102, # Maureen S. O'Brien F train-clean-100 + 544, # Miranda Stinson F train-clean-360 + 653, # cucciasv F train-other-500 + 465, # Leonie Rose F train-clean-100 + 96, # Kymm Zuckert F train-other-500 + 447, # Lee Ann Howlett F train-clean-360 + 165, # Elisabeth Shields F train-clean-100 + 430, # Millbeach F train-other-500 + 214, # Scott Splavec M train-clean-100 + 666, # Kelly Dougherty M train-clean-360 + 481, # Scott Sherris M train-clean-360 + 463, # Chris Hughes M train-other-500 + 273, # Andrew Lebrun M train-other-500 + 172, # Harvey Chinn M train-other-500 + 83, # Graham Williams M train-other-500 + 523, # Michael Loftus M train-clean-360 + 38, # Kurt Copeland M train-clean-360 + 248, # fieldsofgold M train-other-500 + 234, # Menno M train-other-500 + 145, # Mr. Baby Man M train-clean-360 + 250, # Quentin M train-clean-360 + 498, # Chris Gladis M train-clean-100 + 123, # Sean McGaughey M train-clean-360 + 171, # Paul Harvey M train-clean-360 + 49, # Kristen McQuillin F train-clean-100 + 588, # Kalynda F train-clean-360 + 117, # Caitlin Kelly F train-clean-360 + 657, # Shannon F train-other-500 + 275, # Zale Schafer (Rose May Chamberlin Memorial Foundat F train-clean-360 + 604, # Anne-Marie F train-other-500 + 64, # Christiane Levesque F train-clean-360 + 685, # Nikki Sullivan F train-clean-100 + 355, # Lana Taylor F train-clean-100 + 185, # Kim Braun F train-clean-360 + 52, # Cori Samuel F train-other-500 + 218, # Joy Chan F train-other-500 + 549, # AmyAG F train-other-500 + 617, # PJ F train-other-500 + 414, # Christabel F train-clean-100 + 382, # Kelli Robinson F train-clean-360 + 76, # ML Cohen M train-other-500 + 176, # Micah Sheppard M train-clean-360 + 233, # mikenkat M train-clean-360 + 390, # JimmyLogan M train-clean-360 + 393, # Tim Lundeen M train-clean-360 + 425, # RedToby M train-clean-360 + 398, # Sam Fold M train-other-500 + 372, # Jim Mullins M train-clean-360 + 99, # Stewart Wills M train-clean-100 + 340, # Nick Gallant M train-clean-100 + 40, # JemmaBlythe F train-other-500 + 118, # Brenda Dayne F train-clean-360 + 640, # David A. Stokely M train-other-500 + 50, # Dan Threetrees M train-clean-360 + 373, # Brooks Seveer M train-clean-360 + 124, # Steve Karafit M train-clean-100 + 314, # Carl Vonnoh, III M train-clean-360 + 531, # Fr. Richard Zeile of Detroit M train-other-500 + 383, # Mike Roop M train-other-500 + 710, # Sheila Morton F train-clean-100 + 450, # Heather Duncan F train-clean-360 + 645, # Micah M train-other-500 + 517, # Madame Tusk F train-other-500 + 479, # Wina Hathaway F train-other-500 + 30, # Ophelia Darcy F train-other-500 + 220, # Tina Tilney F train-clean-360 + 63, # Linda Wilcox F train-other-500 + 283, # Bethany Simpson F train-clean-360 + 644, # Cynthia Zocca F train-clean-360 + 677, # Allyson Hester F train-other-500 + 21, # Kelly Bescherer F train-other-500 + 552, # Mim Ritty F train-clean-100 + 80, # Fox in the Stars F train-clean-100 + 394, # swroot F train-clean-360 + 426, # Megan Stemm-Wade F train-clean-100 + 91, # Chris Goringe M train-other-500 + 108, # Kevin McAsh M train-clean-360 + 130, # Peter of Buckinghamshire England M train-other-500 + 661, # James Gladwin M train-other-500 + 216, # Dave Ranson M train-clean-100 + 164, # Ed Good M train-other-500 + 308, # Eric Connover M train-other-500 + 569, # Arouet M train-clean-360 + 313, # Tim Bulkeley M train-other-500 + 212, # Glen Hallstrom M train-other-500 + 15, # Chip M train-other-500 + 469, # Christian Pecaut M train-clean-360 + 294, # Diana Kiesners F train-clean-360 + 192, # Nocturna F train-clean-100 + 73, # Claire Goget F train-clean-100 + 417, # Kiki Baessell F train-clean-360 + 636, # Matthew Howell F train-other-500 + 36, # chriss the girl F train-other-500 + 668, # Jan Baxter F train-clean-360 + 403, # Igor Teaforay F train-clean-360 + 618, # Linnea F train-other-500 + 596, # Jo F train-other-500 + 499, # Tammy Sanders F train-clean-100 + 207, # Sage Tyrtle F train-other-500 + 1346, # Jeanie F train-other-500 + 1109, # Martin Geeson M train-other-500 + 770, # Pete Williams, Pittsburgh, PA M train-clean-360 + 1247, # Sarah LuAnn F train-clean-100 + 1526, # Mike Harris M train-other-500 + 908, # Quentin Manuel M train-clean-360 + 1183, # Evelyn Clarke F train-other-500 + 1438, # Tom Barron M train-other-500 + 1022, # peac M train-clean-100 + 1603, # Christine Rodriguez F train-clean-360 + 1425, # Jonah Cummings M train-clean-360 + 731, # Priya, India F train-other-500 + 782, # Alec Daitsman M train-clean-360 + 1090, # Termin Dyan M train-other-500 + 995, # Parrot M train-other-500 + 923, # Jane Greensmith F train-clean-360 + 766, # Clive Catterall M train-other-500 + 822, # kristiface F train-clean-360 + 897, # Jan Dawn Doronila F train-clean-360 + 1579, # Linda Velwest F train-clean-360 + 964, # Utek M train-clean-360 + 1414, # Preston Scrape M train-other-500 + 834, # Serin F train-other-500 + 1302, # davidb M train-clean-360 + 1135, # Linda Andrus F train-clean-360 + 1440, # P Moscato F train-clean-360 + 870, # Barbara Bulkeley F train-clean-360 + 1256, # Graeme Dunlop M train-other-500 + 1255, # Daniel Paashaus M train-other-500 + 1157, # Bev J Stevens F train-clean-360 + 934, # Darla F train-other-500 + 1281, # garbageman99 M train-clean-360 + 819, # n8evv M train-clean-360 + 1041, # mjbrichant F train-other-500 + 863, # K Hindall F train-clean-360 + 1303, # kiwafruit F train-clean-100 + 1115, # Rachel Gatwood F train-clean-360 + 1539, # Nathan Jordan M train-other-500 + 1428, # Gary Dzierlenga M train-other-500 + 1049, # Diana Solomon F train-other-500 + 1546, # Carrie Heyes F train-other-500 + 1089, # Bill Ruhsam M train-clean-360 + 1142, # Jonathan Burchard M train-other-500 + 1375, # Frank Adams M train-clean-360 + 881, # mpetranech M train-other-500 + 798, # Wyatt M train-other-500 + 1647, # Patrick Reinhart M train-clean-360 + 1587, # Claudia Wilson F train-clean-360 + 830, # musici123 F train-other-500 + 1592, # jerryB M train-other-500 + 839, # Ben Dutton M train-other-500 + 835, # Rachel Lintern F train-other-500 + 1273, # gmiteva F train-other-500 + 932, # Raerity F train-other-500 + 1108, # Paul McCartan M train-other-500 + 732, # Tysto M train-clean-360 + 781, # Megan Kunkel F train-other-500 + 1555, # Andrew Nelson M train-clean-360 + 1437, # Charles RUHE M train-clean-360 + 1402, # Angel5 F train-other-500 + 963, # MichelleHarris F train-clean-360 + 1181, # J. Rebecca Franklin F train-clean-360 + 818, # Matt Warzel F train-clean-360 + 1285, # Ric F M train-clean-100 + 797, # Chris Jones F train-other-500 + 1505, # Rom Maczka M train-clean-360 + 1214, # David Baldwin M train-clean-360 + 1636, # jessecoy M train-other-500 + 929, # Petra F train-other-500 + 1171, # Roberta Carlisle F train-other-500 + 817, # texttalker M train-clean-360 + 1433, # browneyedgirl32382 F train-clean-360 + 1158, # StarrDog M train-other-500 + 1000, # artos M train-other-500 + 848, # senshisteph F train-other-500 + 1596, # Joyce Couch F train-other-500 + 757, # Roger Melin M train-clean-360 + 1168, # Epistomolus M train-clean-100 + 741, # Nick Marsh M train-other-500 + 1649, # Phineas Redux M train-other-500 + 851, # Jennifer Lott F train-clean-360 + 808, # M. J. Boyle F train-other-500 + 1595, # Matthew Reece M train-clean-360 + 1370, # Savanna Herrold F train-other-500 + 1565, # bryan.peterson M train-other-500 + 944, # Sarafina Suransky F train-other-500 + 1268, # A. Janelle Risa F train-clean-100 + 771, # Isosceles F train-clean-360 + 752, # Cat Schirf F train-other-500 + 800, # Jack Farrell M train-clean-360 + 1005, # Beatrice F train-other-500 + 1229, # RoseA F train-clean-360 + 943, # Matthew C. Heckel M train-clean-360 + 891, # anoldfashiongirl F train-other-500 + 1226, # serenitylee F train-clean-360 + 1253, # Caroline Shapiro F train-other-500 + 1204, # Dale A. Bade F train-clean-360 + 1230, # Troy Bond M train-other-500 + 791, # David Kleparek M train-clean-100 + 1184, # Joseph Couves F train-other-500 + 1001, # TriciaG F train-clean-360 + 804, # FirstKnight F train-other-500 + 1641, # Kirsten Wever F train-clean-100 + 1259, # Megan Argo F train-other-500 + 1231, # Abigail Bartels F train-other-500 + 1410, # Zachary Johnson M train-other-500 + 1030, # Ancient mariner M train-other-500 + 1093, # Katie Riley F train-clean-360 + 1254, # Rosie F train-clean-100 + 1365, # Eric Leach M train-clean-360 + 831, # David Federman M train-other-500 + 1989, # Joannemmp F train-clean-100 + 1707, # David Olson M train-other-500 + 1849, # Fred DeBerardinis M train-clean-100 + 1808, # Rebecca King F train-clean-360 + 2292, # Arnold M train-clean-100 + 2415, # Patrick Eaton M train-other-500 + 1656, # Sharon Omi F train-clean-100 + 1676, # Gargoyle M train-clean-360 + 1881, # Julienne F train-other-500 + 2036, # T.K. Kirven F train-other-500 + 1761, # EliMarieHK F train-other-500 + 2115, # Pete Milan M train-other-500 + 1803, # Susan Hanfield F train-clean-360 + 1798, # C. L. W. Rollins F train-other-500 + 1723, # Rachel Bossier F train-other-500 + 2341, # Haili F train-other-500 + 2468, # Erin Schellhase F train-clean-360 + 1725, # Ruth Kidson F train-other-500 + 2010, # Peggy F train-other-500 + 1853, # Ron Altman M train-other-500 + 2359, # Doug Reed M train-other-500 + 2422, # Jude Somers F train-clean-360 + 2234, # Coreena F train-other-500 + 2156, # C F de Rosset F train-other-500 + 2483, # Tammy Porter F train-clean-360 + 1781, # humanode M train-clean-360 + 2275, # NatalieOram F train-other-500 + 2390, # sdaeley17 M train-clean-360 + 2314, # Cheri Jordan F train-clean-360 + 2413, # Joanne Rochon F train-clean-360 + 1697, # Lonelle Yoder F train-other-500 + 1718, # Caroline Driggs F train-other-500 + 2387, # Brett G. Hirsch M train-other-500 + 2331, # Madam Fickle F train-clean-100 + 1783, # Sarah Crampton F train-clean-360 + 2397, # Rebecca Braunert-Plunkett F train-other-500 + 2357, # William Gavula M train-other-500 + 1670, # dmbrought M train-other-500 + 1987, # Andrew White M train-clean-360 + 1755, # Yvonne Smith F train-clean-360 + 2192, # Sammy Bean M train-other-500 + 1716, # EyeBones F train-clean-360 + 1828, # David Wales M train-clean-100 + 2251, # Wiley Combs M train-clean-360 + 2065, # Muriel F train-clean-360 + 2017, # CaprishaPage F train-other-500 + 1947, # Barbara Edelman F train-other-500 + 1738, # Lois C. Johnson F train-clean-360 + 1791, # David Cummings M train-clean-360 + 2045, # Linda Ciano F train-clean-360 + 2452, # Walt Allan M train-other-500 + 2040, # MJ Franck F train-other-500 + 1831, # Nigel Boydell M train-other-500 + 2371, # Alexander Hatton M train-clean-360 + 1954, # Szindbad M train-other-500 + 1836, # Kendall Ashyby F train-other-500 + 2436, # josembi M train-other-500 + 2383, # Emma Joyce F train-other-500 + 2278, # Jake Woldstad M train-clean-360 + 1741, # anjieliu F train-other-500 + 1857, # Amanda Friday F train-clean-360 + 2370, # gloriousjob M train-clean-360 + 1907, # Snapdragon F train-other-500 + 2225, # nomorejeffs M train-clean-360 + 2439, # KHand F train-clean-360 + 2239, # amaskill M train-other-500 + 2007, # Art Leung F train-clean-360 + 2283, # Tim Cote M train-clean-360 + 1712, # Steve Belleguelle M train-other-500 + 2094, # Meg Cowan F train-clean-360 + 1772, # haggisreflux M train-clean-360 + 2317, # helengraves F train-clean-360 + 2241, # Steven Reynolds M train-clean-360 + 2011, # pekein M train-clean-360 + 1826, # John Hoerr M train-clean-100 + 1695, # Tina Nuzzi F train-clean-360 + 2451, # DeanOBuchanan M train-clean-100 + 1771, # Chelsea S. F train-other-500 + 2441, # Alison Stewart F train-clean-360 + 1745, # Janet F train-clean-360 + 2358, # Betty Perry F train-clean-360 + 2197, # Mike Nelson M train-other-500 + 2014, # Eden Rea-Hedrick F train-other-500 + 1672, # Mike Wajda M train-clean-360 + 2394, # TinaNygard2 F train-clean-100 + 1657, # alwpoe M train-clean-360 + 1728, # Vinnie Tesla M train-clean-360 + 1805, # Vince Dee M train-clean-100 + 2143, # Suebee F train-clean-360 + 2084, # Eberle Thomas M train-other-500 + 2479, # Daisy Flaim F train-clean-100 + 2152, # Kristel Tretter F train-clean-360 + 2268, # Greg Giordano M train-clean-360 + 1839, # James E. Carson M train-clean-360 + 2056, # acloward M train-clean-360 + 1814, # polkadotish F train-other-500 + 2127, # Ron Lockhart M train-clean-100 + 2114, # Larry Beasley M train-clean-360 + 2469, # Kevin Owens M train-clean-100 + 2447, # Deena Rhoads F train-clean-360 + 1724, # Juliana M. F train-clean-360 + 1869, # NastassiaS F train-other-500 + 2209, # Samantha J Gubitz F train-clean-360 + 2171, # Carolyne F train-other-500 + 2403, # Ian Quinlan M train-clean-360 + 2032, # doonaboon M train-other-500 + 2075, # Joy S Grape F train-clean-360 +] + +# Convert the model speaker IDs back to the dataset speaker IDs +# dataset_speaker_ids: List[int] = [ +# reverse_mapping.get(int(speaker_id)) for speaker_id in selected_speakers +# ] # type: ignore + +# Save the selected speaker IDs +latest_selection: List[int] = [ + 574, + 649, + 102, + 544, + 653, + 666, + 481, + 248, + 123, + 171, + 604, + 64, + 685, + 52, + 218, + 617, + 414, + 425, + 118, + 50, + 373, + 314, + 710, + 450, + 645, + 517, + 63, + 644, + 80, + 394, + 91, + 108, + 661, + 164, + 308, + 469, + 192, + 417, + 668, + 596, + 1109, + 770, + 1247, + 908, + 782, + 995, + 923, + 822, + 1414, + 1302, + 1135, + 1440, + 1281, + 1041, + 1142, + 881, + 835, + 932, + 732, + 1402, + 929, + 817, + 1433, + 1596, + 851, + 1370, + 1204, + 1230, + 791, + 804, + 1808, + 1656, + 2115, + 2341, + 2468, + 1718, + 1783, + 1755, + 2192, + 2371, + 1836, + 1741, + 2439, + 1712, + 2197, + 1728, + 1805, + 2143, + 2084, + 2056, + 2114, + 2447, + 1869, + 2209, +] + +dataset_speaker_ids: List[int] = [ + reverse_mapping.get(int(speaker_id)) for speaker_id in latest_selection +] # type: ignore diff --git a/models/config/stats.json b/models/config/stats.json new file mode 100644 index 0000000000000000000000000000000000000000..6a12abb9aad074831b27a56fe285ce9885d84b0d --- /dev/null +++ b/models/config/stats.json @@ -0,0 +1,3 @@ +{ + "pitch": [85.0, 255.0] +} \ No newline at end of file diff --git a/models/config/symbols.py b/models/config/symbols.py new file mode 100644 index 0000000000000000000000000000000000000000..5793f142e020e98cbba0da1f5a42eb140fd4cef1 --- /dev/null +++ b/models/config/symbols.py @@ -0,0 +1,98 @@ +# NOTE: for the backward comp. +# Prepare the phonemes list and dictionary for the embedding +phoneme_basic_symbols = [ + # IPA symbols + "a", + "b", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "æ", + "ç", + "ð", + "ø", + "ŋ", + "œ", + "ɐ", + "ɑ", + "ɔ", + "ə", + "ɛ", + "ɝ", + "ɹ", + "ɡ", + "ɪ", + "ʁ", + "ʃ", + "ʊ", + "ʌ", + "ʏ", + "ʒ", + "ʔ", + "ˈ", + "ˌ", + "ː", + "̃", + "̍", + "̥", + "̩", + "̯", + "͡", + "θ", + # Punctuation + "!", + "?", + ",", + ".", + "-", + ":", + ";", + '"', + "'", + "(", + ")", + " ", +] + +# TODO: add support for other languages +# _letters_accented = "µßàáâäåæçèéêëìíîïñòóôöùúûüąćęłńœśşźżƒ" +# _letters_cyrilic = "абвгдежзийклмнопрстуфхцчшщъыьэюяёєіїґӧ" +# _pad = "$" + +# This is the list of symbols from StyledTTS2 +_punctuation = ';:,.!?¡¿—…"«»“”' +_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +_letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" + +# Combine all symbols +symbols = list(_punctuation) + list(_letters) + list(_letters_ipa) + +# Add only unique symbols +phones = phoneme_basic_symbols + [ + symbol for symbol in symbols if symbol not in phoneme_basic_symbols +] + +# TODO: Need to understand how to replace this +# len(phones) == 184, leave it as is at this point +symbols = [str(el) for el in range(256)] +symbol2id = {s: i for i, s in enumerate(symbols)} +id2symbol = {i: s for i, s in enumerate(symbols)} diff --git a/models/delightful_hifi.py b/models/delightful_hifi.py new file mode 100644 index 0000000000000000000000000000000000000000..8ac301269b9a5cbcc6a82ddb2c028d93bd954a11 --- /dev/null +++ b/models/delightful_hifi.py @@ -0,0 +1,59 @@ +from lightning.pytorch.core import LightningModule +from torch import Tensor + +from models.config import PreprocessingConfigHifiGAN as PreprocessingConfig +from models.tts.delightful_tts.delightful_tts import DelightfulTTS +from models.vocoder.hifigan import HifiGan + + +class DelightfulHiFi(LightningModule): + def __init__( + self, + delightful_checkpoint_path: str, + hifi_checkpoint_path: str, + lang: str = "en", + sampling_rate: int = 44100, + ): + super().__init__() + + self.sampling_rate = sampling_rate + + self.preprocess_config = PreprocessingConfig( + "multilingual", + sampling_rate=sampling_rate, + ) + + self.delightful_tts = DelightfulTTS.load_from_checkpoint( + delightful_checkpoint_path, + # kwargs to be used in the model + lang=lang, + sampling_rate=sampling_rate, + preprocess_config=self.preprocess_config, + ) + self.delightful_tts.freeze() + + self.hifi_gan = HifiGan.load_from_checkpoint( + hifi_checkpoint_path, + ) + self.hifi_gan.freeze() + + def forward( + self, + text: str, + speaker_idx: Tensor, + ) -> Tensor: + r"""Performs a forward pass through the AcousticModel. + This code must be run only with the loaded weights from the checkpoint! + + Args: + text (str): The input text. + speaker_idx (Tensor): The index of the speaker. + + Returns: + Tensor: The generated waveform with hifi-gan. + """ + mel_pred = self.delightful_tts.forward(text, speaker_idx) + + wav = self.hifi_gan.generator.forward(mel_pred) + + return wav diff --git a/models/delightful_univnet.py b/models/delightful_univnet.py new file mode 100644 index 0000000000000000000000000000000000000000..d8a70929e1e041f569d5e99934cd85107b7412f1 --- /dev/null +++ b/models/delightful_univnet.py @@ -0,0 +1,60 @@ +from lightning.pytorch.core import LightningModule +from torch import Tensor + +from models.config import AcousticENModelConfig +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.tts.delightful_tts.delightful_tts import DelightfulTTS +from models.vocoder.univnet import UnivNet + + +class DelightfulUnivnet(LightningModule): + def __init__( + self, + delightful_checkpoint_path: str, + lang: str = "en", + sampling_rate: int = 22050, + ): + super().__init__() + + self.sampling_rate = sampling_rate + + self.preprocess_config = PreprocessingConfig( + "english_only", + sampling_rate=sampling_rate, + ) + + self.delightful_tts = DelightfulTTS.load_from_checkpoint( + delightful_checkpoint_path, + strict=False, + # kwargs to be used in the model + preprocess_config=self.preprocess_config, + model_config=AcousticENModelConfig(), + lang=lang, + sampling_rate=sampling_rate, + ) + self.delightful_tts.freeze() + + # Don't need to use separated checkpoint, prev checkpoint used + self.univnet = UnivNet() + self.univnet.freeze() + + def forward( + self, + text: str, + speaker_idx: Tensor, + ) -> Tensor: + r"""Performs a forward pass through the AcousticModel. + This code must be run only with the loaded weights from the checkpoint! + + Args: + text (str): The input text. + speaker_idx (Tensor): The index of the speaker. + + Returns: + Tensor: The generated waveform with hifi-gan. + """ + mel_pred = self.delightful_tts.forward(text, speaker_idx) + + wav = self.univnet.forward(mel_pred) + + return wav diff --git a/models/generators/__init__.py b/models/generators/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/generators/delightful_univnet.py b/models/generators/delightful_univnet.py new file mode 100644 index 0000000000000000000000000000000000000000..00d034a858bf8c6b8a98399bdf502b1f6ad7de84 --- /dev/null +++ b/models/generators/delightful_univnet.py @@ -0,0 +1,547 @@ +from typing import List + +from lightning.pytorch.core import LightningModule +import torch +from torch.optim import AdamW, Optimizer, swa_utils +from torch.optim.lr_scheduler import ExponentialLR +from torch.utils.data import DataLoader + +from models.config import ( + AcousticENModelConfig, + AcousticFinetuningConfig, + AcousticPretrainingConfig, + AcousticTrainingConfig, + VocoderFinetuningConfig, + VocoderModelConfig, + VocoderPretrainingConfig, + VoicoderTrainingConfig, + get_lang_map, + lang2id, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.helpers.dataloaders import train_dataloader +from models.helpers.tools import get_mask_from_lengths + +# Models +from models.tts.delightful_tts.acoustic_model import AcousticModel +from models.vocoder.univnet.discriminator import Discriminator +from models.vocoder.univnet.generator import Generator +from training.loss import FastSpeech2LossGen, UnivnetLoss +from training.preprocess.normalize_text import NormalizeText + +# Updated version of the tokenizer +from training.preprocess.tokenizer_ipa_espeak import TokenizerIpaEspeak as TokenizerIPA + + +class DelightfulUnivnet(LightningModule): + r"""DEPRECATED: This idea is basically wrong. The model should synthesis pretty well mel spectrograms and then use them to generate the waveform based on the good quality mel-spec. + + Trainer for the acoustic model. + + Args: + fine_tuning (bool, optional): Whether to use fine-tuning mode or not. Defaults to False. + lang (str): Language of the dataset. + n_speakers (int): Number of speakers in the dataset.generation during training. + batch_size (int): The batch size. + acc_grad_steps (int): The number of gradient accumulation steps. + swa_steps (int): The number of steps for the SWA update. + """ + + def __init__( + self, + fine_tuning: bool = True, + lang: str = "en", + n_speakers: int = 5392, + batch_size: int = 12, + acc_grad_steps: int = 5, + swa_steps: int = 1000, + ): + super().__init__() + + # Switch to manual optimization + self.automatic_optimization = False + self.acc_grad_steps = acc_grad_steps + self.swa_steps = swa_steps + + self.lang = lang + self.fine_tuning = fine_tuning + self.batch_size = batch_size + + lang_map = get_lang_map(lang) + normilize_text_lang = lang_map.nemo + + self.tokenizer = TokenizerIPA(lang) + self.normilize_text = NormalizeText(normilize_text_lang) + + # Acoustic model + self.train_config_acoustic: AcousticTrainingConfig + + if self.fine_tuning: + self.train_config_acoustic = AcousticFinetuningConfig() + else: + self.train_config_acoustic = AcousticPretrainingConfig() + + self.preprocess_config = PreprocessingConfig("english_only") + self.model_config_acoustic = AcousticENModelConfig() + + # TODO: fix the arguments! + self.acoustic_model = AcousticModel( + preprocess_config=self.preprocess_config, + model_config=self.model_config_acoustic, + # NOTE: this parameter may be hyperparameter that you can define based on the demands + n_speakers=n_speakers, + ) + + # Initialize SWA + self.swa_averaged_acoustic = swa_utils.AveragedModel(self.acoustic_model) + + # NOTE: in case of training from 0 bin_warmup should be True! + self.loss_acoustic = FastSpeech2LossGen(bin_warmup=False) + + # Vocoder models + self.model_config_vocoder = VocoderModelConfig() + + self.train_config: VoicoderTrainingConfig = ( + VocoderFinetuningConfig() if fine_tuning else VocoderPretrainingConfig() + ) + + self.univnet = Generator( + model_config=self.model_config_vocoder, + preprocess_config=self.preprocess_config, + ) + self.swa_averaged_univnet = swa_utils.AveragedModel(self.univnet) + + self.discriminator = Discriminator(model_config=self.model_config_vocoder) + self.swa_averaged_discriminator = swa_utils.AveragedModel(self.discriminator) + + self.loss_univnet = UnivnetLoss() + + def forward( + self, text: str, speaker_idx: torch.Tensor, lang: str = "en" + ) -> torch.Tensor: + r"""Performs a forward pass through the AcousticModel. + This code must be run only with the loaded weights from the checkpoint! + + Args: + text (str): The input text. + speaker_idx (torch.Tensor): The index of the speaker. + lang (str): The language. + + Returns: + torch.Tensor: The output of the AcousticModel. + """ + normalized_text = self.normilize_text(text) + _, phones = self.tokenizer(normalized_text) + + # Convert to tensor + x = torch.tensor( + phones, + dtype=torch.int, + device=speaker_idx.device, + ).unsqueeze(0) + + speakers = speaker_idx.repeat(x.shape[1]).unsqueeze(0) + + langs = ( + torch.tensor( + [lang2id[lang]], + dtype=torch.int, + device=speaker_idx.device, + ) + .repeat(x.shape[1]) + .unsqueeze(0) + ) + + y_pred = self.acoustic_model.forward( + x=x, + speakers=speakers, + langs=langs, + ) + + mel_lens = torch.tensor( + [y_pred.shape[2]], + dtype=torch.int32, + device=y_pred.device, + ) + + wav = self.univnet.infer(y_pred, mel_lens) + + return wav + + # TODO: don't forget about torch.no_grad() ! + # default used by the Trainer + # trainer = Trainer(inference_mode=True) + # Use `torch.no_grad` instead + # trainer = Trainer(inference_mode=False) + def training_step(self, batch: List, batch_idx: int): + r"""Performs a training step for the model. + + Args: + batch (List): The batch of data for training. The batch should contain: + - ids: List of indexes. + - raw_texts: Raw text inputs. + - speakers: Speaker identities. + - texts: Text inputs. + - src_lens: Lengths of the source sequences. + - mels: Mel spectrogram targets. + - pitches: Pitch targets. + - pitches_stat: Statistics of the pitches. + - mel_lens: Lengths of the mel spectrograms. + - langs: Language identities. + - attn_priors: Prior attention weights. + - wavs: Waveform targets. + - energies: Energy targets. + batch_idx (int): Index of the batch. + + Returns: + - 'loss': The total loss for the training step. + """ + ( + _, + _, + speakers, + texts, + src_lens, + mels, + pitches, + _, + mel_lens, + langs, + attn_priors, + audio, + energies, + ) = batch + + ##################################### + ## Acoustic model train step ## + ##################################### + + outputs = self.acoustic_model.forward_train( + x=texts, + speakers=speakers, + src_lens=src_lens, + mels=mels, + mel_lens=mel_lens, + pitches=pitches, + langs=langs, + attn_priors=attn_priors, + energies=energies, + ) + + y_pred = outputs["y_pred"] + log_duration_prediction = outputs["log_duration_prediction"] + p_prosody_ref = outputs["p_prosody_ref"] + p_prosody_pred = outputs["p_prosody_pred"] + pitch_prediction = outputs["pitch_prediction"] + energy_pred = outputs["energy_pred"] + energy_target = outputs["energy_target"] + + src_mask = get_mask_from_lengths(src_lens) + mel_mask = get_mask_from_lengths(mel_lens) + + ( + acc_total_loss, + acc_mel_loss, + acc_ssim_loss, + acc_duration_loss, + acc_u_prosody_loss, + acc_p_prosody_loss, + acc_pitch_loss, + acc_ctc_loss, + acc_bin_loss, + acc_energy_loss, + ) = self.loss_acoustic.forward( + src_masks=src_mask, + mel_masks=mel_mask, + mel_targets=mels, + mel_predictions=y_pred, + log_duration_predictions=log_duration_prediction, + u_prosody_ref=outputs["u_prosody_ref"], + u_prosody_pred=outputs["u_prosody_pred"], + p_prosody_ref=p_prosody_ref, + p_prosody_pred=p_prosody_pred, + pitch_predictions=pitch_prediction, + p_targets=outputs["pitch_target"], + durations=outputs["attn_hard_dur"], + attn_logprob=outputs["attn_logprob"], + attn_soft=outputs["attn_soft"], + attn_hard=outputs["attn_hard"], + src_lens=src_lens, + mel_lens=mel_lens, + energy_pred=energy_pred, + energy_target=energy_target, + step=self.trainer.global_step, + ) + + self.log( + "acc_total_loss", acc_total_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_mel_loss", acc_mel_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_ssim_loss", acc_ssim_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_duration_loss", + acc_duration_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "acc_u_prosody_loss", + acc_u_prosody_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "acc_p_prosody_loss", + acc_p_prosody_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "acc_pitch_loss", acc_pitch_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_ctc_loss", acc_ctc_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_bin_loss", acc_bin_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "acc_energy_loss", + acc_energy_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + + ##################################### + ## Univnet model train step ## + ##################################### + fake_audio = self.univnet.forward(y_pred) + + res_fake, period_fake = self.discriminator(fake_audio.detach()) + res_real, period_real = self.discriminator(audio) + + ( + voc_total_loss_gen, + voc_total_loss_disc, + voc_stft_loss, + voc_score_loss, + voc_esr_loss, + voc_snr_loss, + ) = self.loss_univnet.forward( + audio, + fake_audio, + res_fake, + period_fake, + res_real, + period_real, + ) + + self.log( + "voc_total_loss_gen", + voc_total_loss_gen, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "voc_total_loss_disc", + voc_total_loss_disc, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "voc_stft_loss", voc_stft_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "voc_score_loss", voc_score_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "voc_esr_loss", voc_esr_loss, sync_dist=True, batch_size=self.batch_size + ) + self.log( + "voc_snr_loss", voc_snr_loss, sync_dist=True, batch_size=self.batch_size + ) + + # Manual optimizer + # Access your optimizers + optimizers = self.optimizers() + schedulers = self.lr_schedulers() + + #################################### + # Acoustic model manual optimizer ## + #################################### + opt_acoustic: Optimizer = optimizers[0] # type: ignore + sch_acoustic: ExponentialLR = schedulers[0] # type: ignore + + opt_univnet: Optimizer = optimizers[0] # type: ignore + sch_univnet: ExponentialLR = schedulers[0] # type: ignore + + opt_discriminator: Optimizer = optimizers[1] # type: ignore + sch_discriminator: ExponentialLR = schedulers[1] # type: ignore + + # Backward pass for the acoustic model + # NOTE: the loss is divided by the accumulated gradient steps + self.manual_backward(acc_total_loss / self.acc_grad_steps, retain_graph=True) + + # Perform manual optimization univnet + self.manual_backward( + voc_total_loss_gen / self.acc_grad_steps, retain_graph=True + ) + self.manual_backward( + voc_total_loss_disc / self.acc_grad_steps, retain_graph=True + ) + + # accumulate gradients of N batches + if (batch_idx + 1) % self.acc_grad_steps == 0: + # Acoustic model optimizer step + # clip gradients + self.clip_gradients( + opt_acoustic, gradient_clip_val=0.5, gradient_clip_algorithm="norm" + ) + + # optimizer step + opt_acoustic.step() + # Scheduler step + sch_acoustic.step() + # zero the gradients + opt_acoustic.zero_grad() + + # Univnet model optimizer step + # clip gradients + self.clip_gradients( + opt_univnet, gradient_clip_val=0.5, gradient_clip_algorithm="norm" + ) + self.clip_gradients( + opt_discriminator, gradient_clip_val=0.5, gradient_clip_algorithm="norm" + ) + + # optimizer step + opt_univnet.step() + opt_discriminator.step() + + # Scheduler step + sch_univnet.step() + sch_discriminator.step() + + # zero the gradients + opt_univnet.zero_grad() + opt_discriminator.zero_grad() + + # Update SWA model every swa_steps + if self.trainer.global_step % self.swa_steps == 0: + self.swa_averaged_acoustic.update_parameters(self.acoustic_model) + self.swa_averaged_univnet.update_parameters(self.univnet) + self.swa_averaged_discriminator.update_parameters(self.discriminator) + + def on_train_epoch_end(self): + r"""Updates the averaged model after each optimizer step with SWA.""" + self.swa_averaged_acoustic.update_parameters(self.acoustic_model) + self.swa_averaged_univnet.update_parameters(self.univnet) + self.swa_averaged_discriminator.update_parameters(self.discriminator) + + def configure_optimizers(self): + r"""Configures the optimizer used for training. + + Returns + tuple: A tuple containing three dictionaries. Each dictionary contains the optimizer and learning rate scheduler for one of the models. + """ + #################################### + # Acoustic model optimizer config ## + #################################### + # Compute the gamma and initial learning rate based on the current step + lr_decay = self.train_config_acoustic.optimizer_config.lr_decay + default_lr = self.train_config_acoustic.optimizer_config.learning_rate + + init_lr = ( + default_lr + if self.trainer.global_step == 0 + else default_lr * (lr_decay**self.trainer.global_step) + ) + + optimizer_acoustic = AdamW( + self.acoustic_model.parameters(), + lr=init_lr, + betas=self.train_config_acoustic.optimizer_config.betas, + eps=self.train_config_acoustic.optimizer_config.eps, + weight_decay=self.train_config_acoustic.optimizer_config.weight_decay, + ) + + scheduler_acoustic = ExponentialLR(optimizer_acoustic, gamma=lr_decay) + + #################################### + # Univnet model optimizer config ## + #################################### + optim_univnet = AdamW( + self.univnet.parameters(), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_univnet = ExponentialLR( + optim_univnet, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + #################################### + # Discriminator optimizer config ## + #################################### + optim_discriminator = AdamW( + self.discriminator.parameters(), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_discriminator = ExponentialLR( + optim_discriminator, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + return ( + {"optimizer": optimizer_acoustic, "lr_scheduler": scheduler_acoustic}, + {"optimizer": optim_univnet, "lr_scheduler": scheduler_univnet}, + {"optimizer": optim_discriminator, "lr_scheduler": scheduler_discriminator}, + ) + + def on_train_end(self): + # Update SWA models after training + swa_utils.update_bn(self.train_dataloader(), self.swa_averaged_acoustic) + swa_utils.update_bn(self.train_dataloader(), self.swa_averaged_univnet) + swa_utils.update_bn(self.train_dataloader(), self.swa_averaged_discriminator) + + def train_dataloader( + self, + num_workers: int = 5, + root: str = "datasets_cache/LIBRITTS", + cache: bool = True, + cache_dir: str = "datasets_cache", + mem_cache: bool = False, + url: str = "train-960", + ) -> DataLoader: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + num_workers (int): The number of workers. + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. + mem_cache (bool): Whether to use memory cache. + url (str): The URL of the dataset. + + Returns: + Tupple[DataLoader, DataLoader]: The training and validation dataloaders. + """ + return train_dataloader( + batch_size=self.batch_size, + num_workers=num_workers, + root=root, + cache=cache, + cache_dir=cache_dir, + mem_cache=mem_cache, + url=url, + lang=self.lang, + ) diff --git a/models/generators/tests/__init__.py b/models/generators/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/generators/tests/test_delightful_univnet.py b/models/generators/tests/test_delightful_univnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f28ebd4d1c0731f87920af9d738273fe419fd55d --- /dev/null +++ b/models/generators/tests/test_delightful_univnet.py @@ -0,0 +1,30 @@ +# import os +# import unittest + +# from lightning.pytorch import Trainer + +# from models.generators.delightful_univnet import DelightfulUnivnet + +# checkpoint = "checkpoints/logs_new_training_libri-360_energy_epoch=263-step=45639.ckpt" + +# # NOTE: this is needed to avoid CUDA_LAUNCH_BLOCKING error +# os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + +# DEPRECATED +# class TestDelightfulUnivnet(unittest.TestCase): +# def test_train_steps(self): +# default_root_dir = "checkpoints/acoustic" + +# trainer = Trainer( +# default_root_dir=default_root_dir, +# limit_train_batches=1, +# max_epochs=1, +# accelerator="cpu", +# ) + +# module = DelightfulUnivnet(batch_size=1, acc_grad_steps=1, swa_steps=1) + +# train_dataloader = module.train_dataloader(2, cache=False, mem_cache=False) + +# result = trainer.fit(model=module, train_dataloaders=train_dataloader) +# self.assertIsNone(result) diff --git a/models/helpers/__init__.py b/models/helpers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8065744c4e7a9135cc0447250488af155824de2f --- /dev/null +++ b/models/helpers/__init__.py @@ -0,0 +1,2 @@ +from .acoustic import * +from .tools import * diff --git a/models/helpers/acoustic.py b/models/helpers/acoustic.py new file mode 100644 index 0000000000000000000000000000000000000000..471876cbe4f25cf312f99760ee96b02b9cc92168 --- /dev/null +++ b/models/helpers/acoustic.py @@ -0,0 +1,79 @@ +import math + +import torch + + +def positional_encoding( + d_model: int, length: int, +) -> torch.Tensor: + r"""Function to calculate positional encoding for transformer model. + + Args: + d_model (int): Dimension of the model (often corresponds to embedding size). + length (int): Length of sequences. + + Returns: + torch.Tensor: Tensor having positional encodings. + """ + # Initialize placeholder for positional encoding + pe = torch.zeros(length, d_model) + + # Generate position indices and reshape to have shape (length, 1) + position = torch.arange(0, length, dtype=torch.float).unsqueeze(1) + + # Calculate term for division + div_term = torch.exp( + torch.arange(0, d_model, 2).float() + * -(math.log(10000.0) / d_model), + ) + + # Assign sin of position * div_term to even indices in the encoding matrix + pe[:, 0::2] = torch.sin(position * div_term) + + # Assign cos of position * div_term to odd indices in the encoding matrix + pe[:, 1::2] = torch.cos(position * div_term) + + # Add an extra dimension to match expected output shape + return pe.unsqueeze(0) + + +def pitch_phoneme_averaging( + durations: torch.Tensor, + pitches: torch.Tensor, + max_phoneme_len: int) -> torch.Tensor: + r"""Function to compute the average pitch values over the duration of each phoneme. + + Args: + durations (torch.Tensor): Duration of each phoneme for each sample in a batch. + Shape: (batch_size, n_phones) + pitches (torch.Tensor): Per-frame pitch values for each sample in a batch. + Shape: (batch_size, n_mel_timesteps) + max_phoneme_len (int): Maximum length of the phoneme sequence in a batch. + + Returns: + pitches_averaged (torch.Tensor): Tensor containing the averaged pitch values + for each phoneme. Shape: (batch_size, max_phoneme_len) + """ + # Initialize placeholder for averaged pitch values, filling with zeros + pitches_averaged = torch.zeros( + (pitches.shape[0], max_phoneme_len), device=pitches.device, + ) + # Loop over each sample in the batch + for batch_idx in range(durations.shape[0]): + # Set the starting index of pitch sequence + start_idx = 0 + # Loop over each phoneme duration + for i, duration in enumerate(durations[batch_idx]): + # Convert duration to integer + duration = duration.int().item() + # If the duration is not zero + if duration != 0: + # Calculate the mean pitch value for the duration of the current phoneme + mean = torch.mean(pitches[batch_idx, start_idx : start_idx + duration]) + # Store the averaged pitch value + pitches_averaged[batch_idx][i] = mean + # Update the starting index for the next phoneme + start_idx += duration + + # Return tensor with the averaged pitch values + return pitches_averaged diff --git a/models/helpers/dataloaders.py b/models/helpers/dataloaders.py new file mode 100644 index 0000000000000000000000000000000000000000..893a283e45eb76fb9b0c5fccb81f024cba938ebf --- /dev/null +++ b/models/helpers/dataloaders.py @@ -0,0 +1,141 @@ +from typing import List, Optional, Tuple + +from sklearn.model_selection import train_test_split +from torch.utils.data import DataLoader, SequentialSampler + +from training.datasets import LibriTTSDatasetAcoustic + + +def train_dataloader( + batch_size: int = 6, + num_workers: int = 5, + root: str = "datasets_cache/LIBRITTS", + cache: bool = True, + cache_dir: str = "datasets_cache", + mem_cache: bool = False, + url: str = "train-clean-360", + lang: str = "en", + selected_speaker_ids: Optional[List[int]] = None, +) -> DataLoader: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + batch_size (int): The batch size. + num_workers (int): The number of workers. + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. + mem_cache (bool): Whether to use memory cache. + url (str): The URL of the dataset. + lang (str): The language of the dataset. + selected_speaker_ids (Optional[List[int]]): A list of selected speakers. + + Returns: + DataLoader: The training and validation dataloaders. + """ + dataset = LibriTTSDatasetAcoustic( + root=root, + lang=lang, + cache=cache, + cache_dir=cache_dir, + mem_cache=mem_cache, + url=url, + selected_speaker_ids=selected_speaker_ids, + ) + + train_loader = DataLoader( + dataset, + # 4x80Gb max 10 sec audio + # batch_size=20, # self.train_config.batch_size, + # 4*80Gb max ~20.4 sec audio + batch_size=batch_size, + # TODO: find the optimal num_workers + num_workers=num_workers, + persistent_workers=True, + pin_memory=True, + shuffle=False, + collate_fn=dataset.collate_fn, + ) + + return train_loader + + +def train_val_dataloader( + batch_size: int = 6, + num_workers: int = 5, + root: str = "datasets_cache/LIBRITTS", + cache: bool = True, + cache_dir: str = "datasets_cache", + mem_cache: bool = False, + url: str = "train-clean-360", + lang: str = "en", + validation_split: float = 0.02, # Percentage of data to use for validation +) -> Tuple[DataLoader, DataLoader]: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + batch_size (int): The batch size. + num_workers (int): The number of workers. + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. + mem_cache (bool): Whether to use memory cache. + url (str): The URL of the dataset. + lang (str): The language of the dataset. + validation_split (float): The percentage of data to use for validation. + + Returns: + Tupple[DataLoader, DataLoader]: The training and validation dataloaders. + """ + dataset = LibriTTSDatasetAcoustic( + root=root, + lang=lang, + cache=cache, + cache_dir=cache_dir, + mem_cache=mem_cache, + url=url, + ) + + # Split dataset into train and validation + train_indices, val_indices = train_test_split( + list(range(len(dataset))), + test_size=validation_split, + random_state=42, + ) + + # Create Samplers + train_sampler = SequentialSampler(train_indices) + val_sampler = SequentialSampler(val_indices) + + # dataset = LibriTTSMMDatasetAcoustic("checkpoints/libri_preprocessed_data.pt") + train_loader = DataLoader( + dataset, + # 4x80Gb max 10 sec audio + # batch_size=20, # self.train_config.batch_size, + # 4*80Gb max ~20.4 sec audio + batch_size=batch_size, + # TODO: find the optimal num_workers + num_workers=num_workers, + sampler=train_sampler, + persistent_workers=True, + pin_memory=True, + shuffle=False, + collate_fn=dataset.collate_fn, + ) + + val_loader = DataLoader( + dataset, + # 4x80Gb max 10 sec audio + # batch_size=20, # self.train_config.batch_size, + # 4*80Gb max ~20.4 sec audio + batch_size=batch_size, + # TODO: find the optimal num_workers + num_workers=num_workers, + sampler=val_sampler, + persistent_workers=True, + pin_memory=True, + shuffle=False, + collate_fn=dataset.collate_fn, + ) + + return train_loader, val_loader diff --git a/models/helpers/initializer.py b/models/helpers/initializer.py new file mode 100644 index 0000000000000000000000000000000000000000..8d767ecfc360b8faa68f3de0b0206ce2ca655836 --- /dev/null +++ b/models/helpers/initializer.py @@ -0,0 +1,335 @@ +from dataclasses import dataclass +from typing import Tuple + +import torch + +from models.config import ( + SUPPORTED_LANGUAGES, + AcousticENModelConfig, + AcousticModelConfigType, + AcousticPretrainingConfig, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.helpers import positional_encoding, tools +from models.tts.delightful_tts.acoustic_model import AcousticModel +from models.tts.delightful_tts.attention.conformer import Conformer + + +@dataclass +class ConformerConfig: + dim: int + n_layers: int + n_heads: int + embedding_dim: int + p_dropout: float + kernel_size_conv_mod: int + with_ff: bool + + +def get_test_configs( + srink_factor: int = 4, +) -> Tuple[PreprocessingConfig, AcousticENModelConfig, AcousticPretrainingConfig]: + r"""Returns a tuple of configuration objects for testing purposes. + + Args: + srink_factor (int, optional): The shrink factor to apply to the model configuration. Defaults to 4. + + Returns: + Tuple[PreprocessingConfig, AcousticENModelConfig, AcousticPretrainingConfig]: A tuple of configuration objects for testing purposes. + + This function returns a tuple of configuration objects for testing purposes. The configuration objects are as follows: + - `PreprocessingConfig`: A configuration object for preprocessing. + - `AcousticENModelConfig`: A configuration object for the acoustic model. + - `AcousticPretrainingConfig`: A configuration object for acoustic pretraining. + + The `srink_factor` parameter is used to shrink the dimensions of the model configuration to prevent out of memory issues during testing. + """ + preprocess_config = PreprocessingConfig("english_only") + model_config = AcousticENModelConfig() + + model_config.speaker_embed_dim = model_config.speaker_embed_dim // srink_factor + model_config.encoder.n_hidden = model_config.encoder.n_hidden // srink_factor + model_config.decoder.n_hidden = model_config.decoder.n_hidden // srink_factor + model_config.variance_adaptor.n_hidden = ( + model_config.variance_adaptor.n_hidden // srink_factor + ) + + acoustic_pretraining_config = AcousticPretrainingConfig() + + return (preprocess_config, model_config, acoustic_pretraining_config) + + +# Function to initialize a Conformer with a given AcousticModelConfigType configuration +def init_conformer( + model_config: AcousticModelConfigType, +) -> Tuple[Conformer, ConformerConfig]: + r"""Function to initialize a `Conformer` with a given `AcousticModelConfigType` configuration. + + Args: + model_config (AcousticModelConfigType): The object that holds the configuration details. + + Returns: + Conformer: Initialized Conformer object. + + The function sets the details of the `Conformer` object based on the `model_config` parameter. + The `Conformer` configuration is set as follows: + - dim: The number of hidden units, taken from the encoder part of the `model_config.encoder.n_hidden`. + - n_layers: The number of layers, taken from the encoder part of the `model_config.encoder.n_layers`. + - n_heads: The number of attention heads, taken from the encoder part of the `model_config.encoder.n_heads`. + - embedding_dim: The sum of dimensions of speaker embeddings and language embeddings. + The speaker_embed_dim and lang_embed_dim are a part of the `model_config.speaker_embed_dim`. + - p_dropout: Dropout rate taken from the encoder part of the `model_config.encoder.p_dropout`. + It adds a regularization parameter to prevent overfitting. + - kernel_size_conv_mod: The kernel size for the convolution module taken from the encoder part of the `model_config.encoder.kernel_size_conv_mod`. + - with_ff: A Boolean value denoting if feedforward operation is involved, taken from the encoder part of the `model_config.encoder.with_ff`. + + """ + conformer_config = ConformerConfig( + dim=model_config.encoder.n_hidden, + n_layers=model_config.encoder.n_layers, + n_heads=model_config.encoder.n_heads, + embedding_dim=model_config.speaker_embed_dim + + model_config.lang_embed_dim, # speaker_embed_dim + lang_embed_dim = 385 + p_dropout=model_config.encoder.p_dropout, + kernel_size_conv_mod=model_config.encoder.kernel_size_conv_mod, + with_ff=model_config.encoder.with_ff, + ) + + model = Conformer(**vars(conformer_config)) + + return model, conformer_config + + +@dataclass +class AcousticModelConfig: + preprocess_config: PreprocessingConfig + model_config: AcousticENModelConfig + n_speakers: int + + +def init_acoustic_model( + preprocess_config: PreprocessingConfig, + model_config: AcousticENModelConfig, + n_speakers: int = 10, +) -> Tuple[AcousticModel, AcousticModelConfig]: + r"""Function to initialize an `AcousticModel` with given preprocessing and model configurations. + + Args: + preprocess_config (PreprocessingConfig): Configuration object for pre-processing. + model_config (AcousticENModelConfig): Configuration object for English Acoustic model. + n_speakers (int, optional): Number of speakers. Defaults to 10. + + Returns: + AcousticModel: Initialized Acoustic Model. + + The function creates an `AcousticModelConfig` instance which is then used to initialize the `AcousticModel`. + The `AcousticModelConfig` is configured as follows: + - preprocess_config: Pre-processing configuration. + - model_config: English Acoustic model configuration. + - fine_tuning: Boolean flag set to True indicating the model is for fine-tuning. + - n_speakers: Number of speakers. + + """ + # Create an AcousticModelConfig instance + acoustic_model_config = AcousticModelConfig( + preprocess_config=preprocess_config, + model_config=model_config, + n_speakers=n_speakers, + ) + + model = AcousticModel(**vars(acoustic_model_config)) + + return model, acoustic_model_config + + +@dataclass +class ForwardTrainParams: + x: torch.Tensor + speakers: torch.Tensor + src_lens: torch.Tensor + mels: torch.Tensor + mel_lens: torch.Tensor + enc_len: torch.Tensor + pitches: torch.Tensor + pitches_range: Tuple[float, float] + energies: torch.Tensor + langs: torch.Tensor + attn_priors: torch.Tensor + use_ground_truth: bool = True + + +def init_forward_trains_params( + model_config: AcousticENModelConfig, + acoustic_pretraining_config: AcousticPretrainingConfig, + preprocess_config: PreprocessingConfig, + n_speakers: int = 10, +) -> ForwardTrainParams: + r"""Function to initialize the parameters for forward propagation during training. + + Args: + model_config (AcousticENModelConfig): Configuration object for English Acoustic model. + acoustic_pretraining_config (AcousticPretrainingConfig): Configuration object for acoustic pretraining. + preprocess_config (PreprocessingConfig): Configuration object for pre-processing. + n_speakers (int, optional): Number of speakers. Defaults to 10. + + Returns: + ForwardTrainParams: Initialized parameters for forward propagation during training. + + The function initializes the ForwardTrainParams object with the following parameters: + - x: Tensor containing the input sequences. Shape: [speaker_embed_dim, batch_size] + - speakers: Tensor containing the speaker indices. Shape: [speaker_embed_dim, batch_size] + - src_lens: Tensor containing the lengths of source sequences. Shape: [batch_size] + - mels: Tensor containing the mel spectrogram. Shape: [batch_size, speaker_embed_dim, encoder.n_hidden] + - mel_lens: Tensor containing the lengths of mel sequences. Shape: [batch_size] + - pitches: Tensor containing the pitch values. Shape: [batch_size, speaker_embed_dim, encoder.n_hidden] + - energies: Tensor containing the energy values. Shape: [batch_size, speaker_embed_dim, encoder.n_hidden] + - langs: Tensor containing the language indices. Shape: [speaker_embed_dim, batch_size] + - attn_priors: Tensor containing the attention priors. Shape: [batch_size, speaker_embed_dim, speaker_embed_dim] + - use_ground_truth: Boolean flag indicating if ground truth values should be used or not. + + All the Tensors are initialized with random values. + """ + return ForwardTrainParams( + # x: Tensor containing the input sequences. Shape: [speaker_embed_dim, batch_size] + x=torch.randint( + 1, + 255, + ( + model_config.speaker_embed_dim, + acoustic_pretraining_config.batch_size, + ), + ), + pitches_range=(0.0, 1.0), + # speakers: Tensor containing the speaker indices. Shape: [speaker_embed_dim, batch_size] + speakers=torch.randint( + 1, + n_speakers - 1, + ( + model_config.speaker_embed_dim, + acoustic_pretraining_config.batch_size, + ), + ), + # src_lens: Tensor containing the lengths of source sequences. Shape: [speaker_embed_dim] + src_lens=torch.cat( + [ + # torch.tensor([self.model_config.speaker_embed_dim]), + torch.randint( + 1, + acoustic_pretraining_config.batch_size + 1, + (model_config.speaker_embed_dim,), + ), + ], + dim=0, + ), + # mels: Tensor containing the mel spectrogram. Shape: [batch_size, stft.n_mel_channels, encoder.n_hidden] + mels=torch.randn( + model_config.speaker_embed_dim, + preprocess_config.stft.n_mel_channels, + model_config.encoder.n_hidden, + ), + # enc_len: Tensor containing the lengths of mel sequences. Shape: [speaker_embed_dim] + enc_len=torch.cat( + [ + torch.randint( + 1, + model_config.speaker_embed_dim, + (model_config.speaker_embed_dim - 1,), + ), + torch.tensor([model_config.speaker_embed_dim]), + ], + dim=0, + ), + # mel_lens: Tensor containing the lengths of mel sequences. Shape: [batch_size] + mel_lens=torch.cat( + [ + torch.randint( + 1, + model_config.speaker_embed_dim, + (model_config.speaker_embed_dim - 1,), + ), + torch.tensor([model_config.speaker_embed_dim]), + ], + dim=0, + ), + # pitches: Tensor containing the pitch values. Shape: [batch_size, speaker_embed_dim, encoder.n_hidden] + pitches=torch.randn( + # acoustic_pretraining_config.batch_size, + model_config.speaker_embed_dim, + # model_config.speaker_embed_dim, + model_config.encoder.n_hidden, + ), + # energies: Tensor containing the energy values. Shape: [batch_size, speaker_embed_dim, encoder.n_hidden] + energies=torch.randn( + model_config.speaker_embed_dim, + 1, + model_config.encoder.n_hidden, + ), + # langs: Tensor containing the language indices. Shape: [speaker_embed_dim, batch_size] + langs=torch.randint( + 1, + len(SUPPORTED_LANGUAGES) - 1, + ( + model_config.speaker_embed_dim, + acoustic_pretraining_config.batch_size, + ), + ), + # attn_priors: Tensor containing the attention priors. Shape: [batch_size, speaker_embed_dim, speaker_embed_dim] + attn_priors=torch.randn( + model_config.speaker_embed_dim, + model_config.speaker_embed_dim, + acoustic_pretraining_config.batch_size, + ), + use_ground_truth=True, + ) + + +def init_mask_input_embeddings_encoding_attn_mask( + acoustic_model: AcousticModel, + forward_train_params: ForwardTrainParams, + model_config: AcousticENModelConfig, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Function to initialize masks for padding positions, input sequences, embeddings, positional encoding and attention masks. + + Args: + acoustic_model (AcousticModel): Initialized Acoustic Model. + forward_train_params (ForwardTrainParams): Parameters for the forward training process. + model_config (AcousticENModelConfig): Configuration object for English Acoustic model. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing the following elements: + - src_mask: Tensor containing the masks for padding positions in the source sequences. Shape: [1, batch_size] + - x: Tensor containing the input sequences. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim] + - embeddings: Tensor containing the embeddings. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim + lang_embed_dim] + - encoding: Tensor containing the positional encoding. Shape: [lang_embed_dim, max(forward_train_params.mel_lens), model_config.encoder.n_hidden] + - attn_maskЖ Tensor containing the attention masks. Shape: [1, 1, 1, batch_size] + + The function starts by generating masks for padding positions in the source and mel sequences. + Then, it uses the acoustic model to get the input sequences and embeddings. + Finally, it computes the positional encoding. + + """ + # Generate masks for padding positions in the source sequences and mel sequences + # src_mask: Tensor containing the masks for padding positions in the source sequences. Shape: [1, batch_size] + src_mask = tools.get_mask_from_lengths(forward_train_params.src_lens) + + # x: Tensor containing the input sequences. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim] + # embeddings: Tensor containing the embeddings. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim + lang_embed_dim] + x, embeddings = acoustic_model.get_embeddings( + token_idx=forward_train_params.x, + speaker_idx=forward_train_params.speakers, + src_mask=src_mask, + lang_idx=forward_train_params.langs, + ) + + # encoding: Tensor containing the positional encoding + # Shape: [lang_embed_dim, max(forward_train_params.mel_lens), encoder.n_hidden] + encoding = positional_encoding( + model_config.encoder.n_hidden, + max(x.shape[1], int(forward_train_params.mel_lens.max().item())), + ) + + attn_mask = src_mask.view((src_mask.shape[0], 1, 1, src_mask.shape[1])) + + return src_mask, x, embeddings, encoding, attn_mask diff --git a/models/helpers/tests/__init__.py b/models/helpers/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/helpers/tests/test_dataloaders.py b/models/helpers/tests/test_dataloaders.py new file mode 100644 index 0000000000000000000000000000000000000000..5eb2309055125ab286834ce45b67b7d4f7892d33 --- /dev/null +++ b/models/helpers/tests/test_dataloaders.py @@ -0,0 +1,38 @@ +import unittest +from unittest.mock import patch + +from torch.utils.data import DataLoader + +from models.helpers.dataloaders import train_dataloader, train_val_dataloader + + +class TestDataLoader(unittest.TestCase): + def test_train_dataloader(self): + train_loader = train_dataloader( + batch_size=2, + num_workers=2, + cache=False, + mem_cache=False, + ) + + # Assertions + self.assertIsInstance(train_loader, DataLoader) + + for batch in train_loader: + self.assertEqual(len(batch), 13) + break + + def test_train_val_dataloader(self): + train_loader, val_loader = train_val_dataloader( + batch_size=2, + num_workers=2, + cache=False, + mem_cache=False, + ) + + # Assertions + self.assertIsInstance(train_loader, DataLoader) + self.assertIsInstance(val_loader, DataLoader) + +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/test_pitch_phoneme_averaging.py b/models/helpers/tests/test_pitch_phoneme_averaging.py new file mode 100644 index 0000000000000000000000000000000000000000..ace34de1ac88c85acc1fa4e45ee66b4dcb0bce95 --- /dev/null +++ b/models/helpers/tests/test_pitch_phoneme_averaging.py @@ -0,0 +1,36 @@ +import unittest + +import torch + +from models.helpers import ( + pitch_phoneme_averaging, +) + + +class TestPitchPhonemeAveraging(unittest.TestCase): + def test_pitch_phoneme_averaging(self): + # Initialize inputs + durations = torch.tensor([[5, 1, 3, 0], [2, 4, 0, 0]], dtype=torch.float32) + num_phonemes = durations.shape[-1] + max_length = int(torch.sum(durations, dim=1).int().max().item()) + pitches = torch.rand(2, max_length) + + max_phoneme_len = num_phonemes + + # Call the pitch_phoneme_averaging method + result = pitch_phoneme_averaging(durations, pitches, max_phoneme_len) + + # Assert output type + self.assertIsInstance(result, torch.Tensor) + + # Assert output shape + expected_shape = durations.shape + self.assertEqual(result.shape, expected_shape) + + # Assert all pitch values are within [0,1] since input pitch values were from a uniform distribution over [0,1] + self.assertTrue(torch.all((result >= 0) & (result <= 1))) + + +# Run tests +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/test_position_encoding.py b/models/helpers/tests/test_position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..2c0d784177241acffc4c17690a34f4f369ba869b --- /dev/null +++ b/models/helpers/tests/test_position_encoding.py @@ -0,0 +1,30 @@ +# Required Libraries +import unittest + +import torch + +from models.helpers import positional_encoding + + +class TestPositionalEncoding(unittest.TestCase): + def test_positional_encoding(self): + # Test with d_model=128, length=10 + d_model = 128 + length = 10 + result = positional_encoding(d_model, length) + + # Assert that output is a torch.Tensor + self.assertIsInstance(result, torch.Tensor) + + # Assert the output tensor shape is correct + # The extra dimension from unsqueeze is considered + expected_shape = (1, length, d_model) + self.assertEqual(result.shape, expected_shape) + + # Assert that values lie in the range [-1, 1] + self.assertTrue(torch.all((result >= -1) & (result <= 1))) + + +# Run tests +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/tests_tools/__init__.py b/models/helpers/tests/tests_tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/helpers/tests/tests_tools/test_calc_same_padding.py b/models/helpers/tests/tests_tools/test_calc_same_padding.py new file mode 100644 index 0000000000000000000000000000000000000000..0c8b80f381f012cb69a7d92f6b193cd0a2ae5574 --- /dev/null +++ b/models/helpers/tests/tests_tools/test_calc_same_padding.py @@ -0,0 +1,34 @@ +import unittest + +from models.helpers.tools import calc_same_padding + + +class TestCalcSamePadding(unittest.TestCase): + def test_odd_kernel_size(self): + """Test that the padding returns correct tuple for odd-sized kernel.""" + kernel_size = 3 # an odd-sized kernel + pad = calc_same_padding(kernel_size) + self.assertEqual(pad, (1, 1)) # we expect padding of size 1 on both sides + + def test_even_kernel_size(self): + """Test that the padding returns correct tuple for even-sized kernel.""" + kernel_size = 4 # an even-sized kernel + pad = calc_same_padding(kernel_size) + self.assertEqual( + pad, (2, 1), + ) # we expect padding of size 2 on one side, and size 1 on the other side + + def test_one_kernel_size(self): + """Test that the padding returns correct tuple for kernel of size 1.""" + kernel_size = 1 # kernel of size 1 + pad = calc_same_padding(kernel_size) + self.assertEqual(pad, (0, 0)) # we expect no padding on both sides + + def test_zero_kernel_size(self): + """Test that the padding raises ValueError for invalid kernel size of zero.""" + with self.assertRaises(ValueError): + calc_same_padding(0) # a kernel of size 0 is not valid + + +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/tests_tools/test_get_mask_from_lengths.py b/models/helpers/tests/tests_tools/test_get_mask_from_lengths.py new file mode 100644 index 0000000000000000000000000000000000000000..b270c731bf9ea3e5610e9f1cb7428b861d40a3ee --- /dev/null +++ b/models/helpers/tests/tests_tools/test_get_mask_from_lengths.py @@ -0,0 +1,37 @@ +import unittest + +import torch + +from models.helpers.tools import get_mask_from_lengths + + +class TestGetMaskFromLengths(unittest.TestCase): + def setUp(self): + # Test cases: [2, 3, 1, 4] + self.input_lengths = torch.tensor([2, 3, 1, 4]) + + def test_get_mask_from_lengths(self): + expected_output = torch.tensor( + [ + [False, False, True, True], + [False, False, False, True], + [False, True, True, True], + [False, False, False, False], + ], + ) + + # Call the function with the test cases + output = get_mask_from_lengths(self.input_lengths) + + # Check if the output is a tensor + self.assertIsInstance(output, torch.Tensor) + + # Check if the output shape is as expected + self.assertEqual(output.shape, expected_output.shape) + + # Check if the output values are as expected + self.assertTrue(torch.all(output.eq(expected_output))) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/tests_tools/test_initialize_embeddings.py b/models/helpers/tests/tests_tools/test_initialize_embeddings.py new file mode 100644 index 0000000000000000000000000000000000000000..b04514a77fe39ab2da8ecad2406cd16e137d43b2 --- /dev/null +++ b/models/helpers/tests/tests_tools/test_initialize_embeddings.py @@ -0,0 +1,45 @@ +import unittest + +import numpy as np +import torch + +from models.helpers.tools import initialize_embeddings + + +class TestInitializeEmbeddings(unittest.TestCase): + def test_initialize_embeddings(self): + # Test with correct input shape + shape = (5, 10) + result = initialize_embeddings(shape) + # Assert output is torch.Tensor + self.assertIsInstance(result, torch.Tensor) + # Assert output shape + self.assertEqual(result.shape, shape) + # Assert type of elements + self.assertEqual(result.dtype, torch.float32) + + # Assert standard deviation is close to expected (within some tolerance) + expected_stddev = np.sqrt(2 / shape[1]) + tolerance = 0.1 + self.assertLessEqual(abs(result.std().item() - expected_stddev), tolerance) + + # Test with incorrect number of dimensions in shape + incorrect_shape = (5, 10, 15) + with self.assertRaises(AssertionError) as context: + initialize_embeddings(incorrect_shape) + self.assertEqual( + str(context.exception), "Can only initialize 2-D embedding matrices ...", + ) + + # Test with zero dimensions in shape + zero_dim_shape = () + with self.assertRaises(AssertionError) as context: + initialize_embeddings(zero_dim_shape) + self.assertEqual( + str(context.exception), "Can only initialize 2-D embedding matrices ...", + ) + + +# Run tests +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/tests_tools/test_pad.py b/models/helpers/tests/tests_tools/test_pad.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a3052cc70c117f8d9f08dd31614e68a3c95b3a --- /dev/null +++ b/models/helpers/tests/tests_tools/test_pad.py @@ -0,0 +1,36 @@ +import unittest + +import torch + +from models.helpers.tools import pad + + +class TestPad(unittest.TestCase): + def test_1D_tensors_pad(self): + # 1D tensor inputs + tensors = [torch.ones(n) for n in range(1, 11)] + # Ten 1D tensors of length 1 to 10 + max_len = max(t.numel() for t in tensors) + + # Call the function + result = pad(tensors, max_len) + + # Check the output shape is as expected + self.assertTrue(all(t.numel() == max_len for t in result)) + + def test_2D_tensors_pad(self): + # 2D tensor inputs + tensors = [torch.ones(n, 5) for n in range(1, 11)] + max_len = max(t.size(0) for t in tensors) + + # Call the function + result = pad(tensors, max_len) + + # Check the output shape is as expected + self.assertTrue(all(t.size(0) == max_len for t in result)) + # Make sure second dimension wasn't changed + self.assertTrue(all(t.size(1) == 5 for t in result)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tests/tests_tools/test_stride_lens_downsampling.py b/models/helpers/tests/tests_tools/test_stride_lens_downsampling.py new file mode 100644 index 0000000000000000000000000000000000000000..41d8c1bf32ba04f32c20b56cb22af77d433529b1 --- /dev/null +++ b/models/helpers/tests/tests_tools/test_stride_lens_downsampling.py @@ -0,0 +1,52 @@ +import unittest + +import torch + +from models.helpers import ( + stride_lens_downsampling, +) + + +class TestStrideLens(unittest.TestCase): + def test_stride_lens(self): + # Define test case inputs + input_lengths = torch.tensor([5, 7, 10, 12]) + stride = 2 + + # Correct output for this would be ceil([5, 7, 10, 12] / 2) => [3, 4, 5, 6] + expected_output = torch.tensor([3, 4, 5, 6]) + + # Call the function with the test cases + output = stride_lens_downsampling(input_lengths, stride) + + # Check if the output is a tensor + self.assertIsInstance(output, torch.Tensor) + + # Check if the output shape is as expected + self.assertEqual(output.shape, expected_output.shape) + + # Check if the output values are as expected + self.assertTrue(torch.all(output.eq(expected_output))) + + def test_stride_lens_default_stride(self): + # Define test case inputs. Here, we do not provide the stride. + input_lengths = torch.tensor([10, 20, 4, 11]) + + # Correct output for this would be ceil([10, 20, 4, 11] / 2) => [5, 10, 2, 6] + expected_output = torch.tensor([5, 10, 2, 6]) + + # Call the function with the test cases + output = stride_lens_downsampling(input_lengths) + + # Check if the output is a tensor + self.assertIsInstance(output, torch.Tensor) + + # Check if the output shape is as expected + self.assertEqual(output.shape, expected_output.shape) + + # Check if the output values are as expected + self.assertTrue(torch.all(output.eq(expected_output))) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/helpers/tools.py b/models/helpers/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..33cc1fda447705373dee32c2f5b155629e63f9e7 --- /dev/null +++ b/models/helpers/tools.py @@ -0,0 +1,157 @@ +from typing import List, Tuple + +import numpy as np +import torch +import torch.nn.functional as F + + +def get_device() -> torch.device: + r"""Function returns the device where the model and tensors should be placed. + + Returns + torch.device: The device where the model and tensors should be placed. + """ + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def pad(input_ele: List[torch.Tensor], max_len: int) -> torch.Tensor: + r"""Takes a list of 1D or 2D tensors and pads them to match the maximum length. + + Args: + input_ele (List[torch.Tensor]): The list of tensors to be padded. + max_len (int): The length to which the tensors should be padded. + + Returns: + torch.Tensor: A tensor containing all the padded input tensors. + """ + # Create an empty list to store the padded tensors + out_list = torch.jit.annotate(List[torch.Tensor], []) + for batch in input_ele: + if len(batch.shape) == 1: + # Perform padding for 1D tensor + one_batch_padded = F.pad( + batch, + (0, max_len - batch.size(0)), + "constant", + 0.0, + ) + else: + # Perform padding for 2D tensor + one_batch_padded = F.pad( + batch, + (0, 0, 0, max_len - batch.size(0)), + "constant", + 0.0, + ) + # Append the padded tensor to the list + out_list.append(one_batch_padded) + + # Stack all the tensors in the list into a single tensor + return torch.stack(out_list) + + +def get_mask_from_lengths(lengths: torch.Tensor) -> torch.Tensor: + r"""Generate a mask tensor from a tensor of sequence lengths. + + Args: + lengths (torch.Tensor): A tensor of sequence lengths of shape: (batch_size, ) + + Returns: + torch.Tensor: A mask tensor of shape: (batch_size, max_len) where max_len is the + maximum sequence length in the provided tensor. The mask tensor has a value of + True at each position that is more than the length of the sequence (padding positions). + + Example: + lengths: `torch.tensor([2, 3, 1, 4])` + Mask tensor will be: `torch.tensor([ + [False, False, True, True], + [False, False, False, True], + [False, True, True, True], + [False, False, False, False] + ])` + """ + # Get batch size + batch_size = lengths.shape[0] + + # Get maximum sequence length in the batch + max_len = int(torch.max(lengths).item()) + + # Generate a tensor of shape (batch_size, max_len) + # where each row contains values from 0 to max_len + ids = ( + torch.arange(0, max_len, device=lengths.device) + .unsqueeze(0) + .expand(batch_size, -1) + ) + # Compare each value in the ids tensor with + # corresponding sequence length to generate a mask. + # The mask will have True at positions where id >= sequence length, + # indicating padding positions in the original sequences + return ids >= lengths.unsqueeze(1).type(torch.int64).expand(-1, max_len) + + +def stride_lens_downsampling(lens: torch.Tensor, stride: int = 2) -> torch.Tensor: + r"""Function computes the lengths of 1D tensor when applying a stride for downsampling. + + Args: + lens (torch.Tensor): Tensor containing the lengths to be downsampled. + stride (int, optional): The stride to be used for downsampling. Defaults to 2. + + Returns: + torch.Tensor: A tensor of the same shape as the input containing the downsampled lengths. + """ + # The torch.ceil function is used to handle cases where the length is not evenly divisible + # by the stride. The torch.ceil function rounds up to the nearest integer, ensuring that + # each item is present at least once in the downsampled lengths. + # Finally, the .int() is used to convert the resulting float32 tensor to an integer tensor. + return torch.ceil(lens / stride).int() + + +def calc_same_padding(kernel_size: int) -> Tuple[int, int]: + r"""Calculates the necessary padding for 'same' padding in convolutional operations. + + For 'same' padding, the output size is the same as the input size for `stride=1`. This function returns + two integers, representing the padding to be added on either side of the input to achieve 'same' padding. + + Args: + kernel_size (int): Size of the convolving kernel. + + Returns: + Tuple[int, int]: A tuple of two integers representing the number of padding elements to be applied on + left and right (or top and bottom for 2D) of the input tensor respectively. + """ + # Check if kernel_size is an integer greater than zero + if not isinstance(kernel_size, int) or kernel_size <= 0: + raise ValueError("kernel_size must be an integer greater than zero") + + # Determine base padding amount (equal to half the kernel size, truncated down) + pad = kernel_size // 2 + + # Return padding for each side of the kernel. If kernel size is odd, padding is (pad, pad). + # If kernel size is even, padding is (pad, pad - 1) because we can't pad equally on both sides. + return (pad, pad - (kernel_size + 1) % 2) + + +def initialize_embeddings(shape: Tuple[int, ...]) -> torch.Tensor: + r"""Initialize embeddings using Kaiming initialization (He initialization). + + This method is specifically designed for 2D matrices and helps to avoid + the vanishing/exploding gradient problem in deep neural networks. + This is achieved by keeping the variance of the outputs of a layer to be + the same as the variance of its inputs. + + Args: + shape (Tuple[int, ...]): The shape of the embedding matrix to create, denoted as a tuple of integers. + The shape should comprise 2 dimensions, i.e., (embedding_dim, num_embeddings). + + Raises: + AssertionError: if the provided shape is not 2D. + + Returns: + torch.Tensor: the created embedding matrix. + """ + # Check if the input shape is 2D + assert len(shape) == 2, "Can only initialize 2-D embedding matrices ..." + + # Initialize the embedding matrix using Kaiming initialization + return torch.randn(shape) * np.sqrt(2 / shape[1]) diff --git a/models/tts/__init__.py b/models/tts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c690fa2ee15397bd816d7710e192cc0e594824 --- /dev/null +++ b/models/tts/__init__.py @@ -0,0 +1 @@ +from .delightful_tts import DelightfulTTS diff --git a/models/tts/delightful_tts/__init__.py b/models/tts/delightful_tts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c690fa2ee15397bd816d7710e192cc0e594824 --- /dev/null +++ b/models/tts/delightful_tts/__init__.py @@ -0,0 +1 @@ +from .delightful_tts import DelightfulTTS diff --git a/models/tts/delightful_tts/acoustic_model/__init__.py b/models/tts/delightful_tts/acoustic_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2f73e3439ddfcfc94ecb8ad432da270eb9c8d656 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/__init__.py @@ -0,0 +1,6 @@ +from .acoustic_model import AcousticModel +from .embedding import Embedding +from .length_adaptor import LengthAdaptor +from .phoneme_prosody_predictor import PhonemeProsodyPredictor +from .pitch_adaptor_conv import PitchAdaptorConv +from .variance_predictor import VariancePredictor diff --git a/models/tts/delightful_tts/acoustic_model/acoustic_model.py b/models/tts/delightful_tts/acoustic_model/acoustic_model.py new file mode 100644 index 0000000000000000000000000000000000000000..41521f59e96f7a632d36848f56476abc49be9033 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/acoustic_model.py @@ -0,0 +1,547 @@ +from typing import Dict, Tuple, Union + +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from models.config import ( + SUPPORTED_LANGUAGES, + AcousticModelConfigType, + PreprocessingConfig, + symbols, +) +from models.helpers import ( + positional_encoding, + tools, +) +from models.tts.delightful_tts.attention import Conformer +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.reference_encoder import ( + PhonemeLevelProsodyEncoder, + UtteranceLevelProsodyEncoder, +) + +from .aligner import Aligner +from .energy_adaptor import EnergyAdaptor +from .length_adaptor import LengthAdaptor +from .phoneme_prosody_predictor import PhonemeProsodyPredictor +from .pitch_adaptor_conv import PitchAdaptorConv + + +class AcousticModel(Module): + r"""The DelightfulTTS AcousticModel class represents a PyTorch module for an acoustic model in text-to-speech (TTS). + The acoustic model is responsible for predicting speech signals from phoneme sequences. + + The model comprises multiple sub-modules including encoder, decoder and various prosody encoders and predictors. + Additionally, a pitch and length adaptor are instantiated. + + Args: + preprocess_config (PreprocessingConfig): Object containing the configuration used for preprocessing the data + model_config (AcousticModelConfigType): Configuration object containing various model parameters + n_speakers (int): Total number of speakers in the dataset + leaky_relu_slope (float, optional): Slope for the leaky relu. Defaults to LEAKY_RELU_SLOPE. + + Note: + For more specific details on the implementation of sub-modules please refer to their individual respective modules. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType, + n_speakers: int, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + self.emb_dim = model_config.encoder.n_hidden + + self.encoder = Conformer( + dim=model_config.encoder.n_hidden, + n_layers=model_config.encoder.n_layers, + n_heads=model_config.encoder.n_heads, + embedding_dim=model_config.speaker_embed_dim + model_config.lang_embed_dim, + p_dropout=model_config.encoder.p_dropout, + kernel_size_conv_mod=model_config.encoder.kernel_size_conv_mod, + with_ff=model_config.encoder.with_ff, + ) + + self.pitch_adaptor_conv = PitchAdaptorConv( + channels_in=model_config.encoder.n_hidden, + channels_hidden=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + emb_kernel_size=model_config.variance_adaptor.emb_kernel_size, + dropout=model_config.variance_adaptor.p_dropout, + leaky_relu_slope=leaky_relu_slope, + ) + + self.energy_adaptor = EnergyAdaptor( + channels_in=model_config.encoder.n_hidden, + channels_hidden=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + emb_kernel_size=model_config.variance_adaptor.emb_kernel_size, + dropout=model_config.variance_adaptor.p_dropout, + leaky_relu_slope=leaky_relu_slope, + ) + + self.length_regulator = LengthAdaptor(model_config) + + self.utterance_prosody_encoder = UtteranceLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.utterance_prosody_predictor = PhonemeProsodyPredictor( + model_config=model_config, + phoneme_level=False, + ) + + self.phoneme_prosody_encoder = PhonemeLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.phoneme_prosody_predictor = PhonemeProsodyPredictor( + model_config=model_config, + phoneme_level=True, + ) + + self.u_bottle_out = nn.Linear( + model_config.reference_encoder.bottleneck_size_u, + model_config.encoder.n_hidden, + ) + + self.u_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_u, + elementwise_affine=False, + ) + + self.p_bottle_out = nn.Linear( + model_config.reference_encoder.bottleneck_size_p, + model_config.encoder.n_hidden, + ) + + self.p_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_p, + elementwise_affine=False, + ) + + self.aligner = Aligner( + d_enc_in=model_config.encoder.n_hidden, + d_dec_in=preprocess_config.stft.n_mel_channels, + d_hidden=model_config.encoder.n_hidden, + ) + + self.decoder = Conformer( + dim=model_config.decoder.n_hidden, + n_layers=model_config.decoder.n_layers, + n_heads=model_config.decoder.n_heads, + embedding_dim=model_config.speaker_embed_dim + model_config.lang_embed_dim, + p_dropout=model_config.decoder.p_dropout, + kernel_size_conv_mod=model_config.decoder.kernel_size_conv_mod, + with_ff=model_config.decoder.with_ff, + ) + + self.src_word_emb = Parameter( + tools.initialize_embeddings( + (len(symbols), model_config.encoder.n_hidden), + ), + ) + + self.to_mel = nn.Linear( + model_config.decoder.n_hidden, + preprocess_config.stft.n_mel_channels, + ) + + # NOTE: here you can manage the speaker embeddings, can be used for the voice export ? + # NOTE: flexibility of the model binded by the n_speaker parameter, maybe I can find another way? + # NOTE: in LIBRITTS there are 2477 speakers, we can add more, just extend the speaker_embed matrix + # Need to think about it more + self.speaker_embed = Parameter( + tools.initialize_embeddings( + (n_speakers, model_config.speaker_embed_dim), + ), + ) + + self.lang_embed = Parameter( + tools.initialize_embeddings( + (len(SUPPORTED_LANGUAGES), model_config.lang_embed_dim), + ), + ) + + def get_embeddings( + self, + token_idx: torch.Tensor, + speaker_idx: torch.Tensor, + src_mask: torch.Tensor, + lang_idx: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Given the tokens, speakers, source mask, and language indices, compute + the embeddings for tokens, speakers and languages and return the + token_embeddings and combined speaker and language embeddings + + Args: + token_idx (torch.Tensor): Tensor of token indices. + speaker_idx (torch.Tensor): Tensor of speaker identities. + src_mask (torch.Tensor): Mask tensor for source sequences. + lang_idx (torch.Tensor): Tensor of language indices. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Token embeddings tensor, + and combined speaker and language embeddings tensor. + """ + token_embeddings = F.embedding(token_idx, self.src_word_emb) + # NOTE: here you can manage the speaker embeddings, can be used for the voice export ? + speaker_embeds = F.embedding(speaker_idx, self.speaker_embed) + lang_embeds = F.embedding(lang_idx, self.lang_embed) + + # Merge the speaker and language embeddings + embeddings = torch.cat([speaker_embeds, lang_embeds], dim=2) + + # Apply the mask to the embeddings and token embeddings + embeddings = embeddings.masked_fill(src_mask.unsqueeze(-1), 0.0) + token_embeddings = token_embeddings.masked_fill(src_mask.unsqueeze(-1), 0.0) + + return token_embeddings, embeddings + + def prepare_for_export(self) -> None: + r"""Prepare the model for export. + + This method is called when the model is about to be exported, such as for deployment + or serializing for later use. The method removes unnecessary components that are + not needed during inference. Specifically, it removes the phoneme and utterance + prosody encoders for this acoustic model. These components are typically used during + training and are not needed when the model is used for making predictions. + + Returns + None + """ + del self.phoneme_prosody_encoder + del self.utterance_prosody_encoder + + # NOTE: freeze/unfreeze params changed, because of the conflict with the lightning module + def freeze_params(self) -> None: + r"""Freeze the trainable parameters in the model. + + By freezing, the parameters are no longer updated by gradient descent. + This is typically done when you want to keep parts of your model fixed while training other parts. + For this model, it freezes all parameters and then selectively unfreezes the + speaker embeddings and the pitch adaptor's pitch embeddings to allow these components to update during training. + + Returns + None + """ + for par in self.parameters(): + par.requires_grad = False + self.speaker_embed.requires_grad = True + + # NOTE: freeze/unfreeze params changed, because of the conflict with the lightning module + def unfreeze_params(self, freeze_text_embed: bool, freeze_lang_embed: bool) -> None: + r"""Unfreeze the trainable parameters in the model, allowing them to be updated during training. + + This method is typically used to 'unfreeze' previously 'frozen' parameters, making them trainable again. + For this model, it unfreezes all parameters and then selectively freezes the + text embeddings and language embeddings, if required. + + Args: + freeze_text_embed (bool): Flag to indicate if text embeddings should remain frozen. + freeze_lang_embed (bool): Flag to indicate if language embeddings should remain frozen. + + Returns: + None + """ + # Iterate through all model parameters and make them trainable + for par in self.parameters(): + par.requires_grad = True + + # If freeze_text_embed flag is True, keep the source word embeddings frozen + if freeze_text_embed: + # @fixed self.src_word_emb.parameters has no parameters() method! + # for par in self.src_word_emb.parameters(): + self.src_word_emb.requires_grad = False + + # If freeze_lang_embed flag is True, keep the language embeddings frozen + if freeze_lang_embed: + self.lang_embed.requires_grad = False + + def average_utterance_prosody( + self, + u_prosody_pred: torch.Tensor, + src_mask: torch.Tensor, + ) -> torch.Tensor: + r"""Compute the average utterance prosody over the length of non-masked elements. + + This method averages the output of the utterance prosody predictor over + the sequence lengths (non-masked elements). This function will return + a tensor with the same first dimension but singleton trailing dimensions. + + Args: + u_prosody_pred (torch.Tensor): Tensor containing the predicted utterance prosody of dimension (batch_size, T, n_features). + src_mask (torch.Tensor): Tensor of dimension (batch_size, T) acting as a mask where masked entries are set to False. + + Returns: + torch.Tensor: Tensor of dimension (batch_size, 1, n_features) containing average utterance prosody over non-masked sequence length. + """ + # Compute the real sequence lengths by negating the mask and summing along the sequence dimension + lengths = ((~src_mask) * 1.0).sum(1) + + # Compute the sum of u_prosody_pred across the sequence length dimension, + # then divide by the sequence lengths tensor to calculate the average. + # This performs a broadcasting operation to account for the third dimension (n_features). + # Return the averaged prosody prediction + return u_prosody_pred.sum(1, keepdim=True) / lengths.view(-1, 1, 1) + + def forward_train( + self, + x: torch.Tensor, + speakers: torch.Tensor, + src_lens: torch.Tensor, + mels: torch.Tensor, + mel_lens: torch.Tensor, + pitches: torch.Tensor, + langs: torch.Tensor, + attn_priors: Union[torch.Tensor, None], + energies: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + r"""Forward pass during training phase. + + For a given phoneme sequence, speaker identities, sequence lengths, mels, + mel lengths, pitches, language, and attention priors, the forward pass + processes these inputs through the defined architecture. + + Args: + x (torch.Tensor): Tensor of phoneme sequence. + speakers (torch.Tensor): Tensor of speaker identities. + src_lens (torch.Tensor): Long tensor representing the lengths of source sequences. + mels (torch.Tensor): Tensor of mel spectrograms. + mel_lens (torch.Tensor): Long tensor representing the lengths of mel sequences. + pitches (torch.Tensor): Tensor of pitch values. + langs (torch.Tensor): Tensor of language identities. + attn_priors (torch.Tensor): Prior attention values. + energies (torch.Tensor): Tensor of energy values. + + Returns: + Dict[str, torch.Tensor]: Returns the prediction outputs as a dictionary. + """ + # Generate masks for padding positions in the source sequences and mel sequences + src_mask = tools.get_mask_from_lengths(src_lens) + mel_mask = tools.get_mask_from_lengths(mel_lens) + + x, embeddings = self.get_embeddings( + token_idx=x, + speaker_idx=speakers, + src_mask=src_mask, + lang_idx=langs, + ) + + encoding = positional_encoding( + self.emb_dim, + max(x.shape[1], int(mel_lens.max().item())), + ) + x = x.to(src_mask.device) + encoding = encoding.to(src_mask.device) + embeddings = embeddings.to(src_mask.device) + + x = self.encoder(x, src_mask, embeddings=embeddings, encoding=encoding) + + u_prosody_ref = self.u_norm( + self.utterance_prosody_encoder(mels=mels, mel_lens=mel_lens), + ) + u_prosody_pred = self.u_norm( + self.average_utterance_prosody( + u_prosody_pred=self.utterance_prosody_predictor(x=x, mask=src_mask), + src_mask=src_mask, + ), + ) + + p_prosody_ref = self.p_norm( + self.phoneme_prosody_encoder( + x=x, + src_mask=src_mask, + mels=mels, + mel_lens=mel_lens, + encoding=encoding, + ), + ) + p_prosody_pred = self.p_norm( + self.phoneme_prosody_predictor( + x=x, + mask=src_mask, + ), + ) + + x = x + self.u_bottle_out(u_prosody_pred) + x = x + self.p_bottle_out(p_prosody_pred) + + # Save the residual for later use + x_res = x + + attn_logprob, attn_soft, attn_hard, attn_hard_dur = self.aligner( + enc_in=x_res.permute((0, 2, 1)), + dec_in=mels, + enc_len=src_lens, + dec_len=mel_lens, + enc_mask=src_mask, + attn_prior=attn_priors, + ) + + attn_hard_dur = attn_hard_dur.to(src_mask.device) + + x, pitch_prediction, avg_pitch_target = ( + self.pitch_adaptor_conv.add_pitch_embedding_train( + x=x, + target=pitches, + dr=attn_hard_dur, + mask=src_mask, + ) + ) + + energies = energies.to(src_mask.device) + + x, energy_pred, avg_energy_target = ( + self.energy_adaptor.add_energy_embedding_train( + x=x, + target=energies, + dr=attn_hard_dur, + mask=src_mask, + ) + ) + + x, log_duration_prediction, embeddings = self.length_regulator.upsample_train( + x=x, + x_res=x_res, + duration_target=attn_hard_dur, + src_mask=src_mask, + embeddings=embeddings, + ) + + # Decode the encoder output to pred mel spectrogram + decoder_output = self.decoder( + x, + mel_mask, + embeddings=embeddings, + encoding=encoding, + ) + + y_pred = self.to_mel(decoder_output) + y_pred = y_pred.permute((0, 2, 1)) + + return { + "y_pred": y_pred, + "pitch_prediction": pitch_prediction, + "pitch_target": avg_pitch_target, + "energy_pred": energy_pred, + "energy_target": avg_energy_target, + "log_duration_prediction": log_duration_prediction, + "u_prosody_pred": u_prosody_pred, + "u_prosody_ref": u_prosody_ref, + "p_prosody_pred": p_prosody_pred, + "p_prosody_ref": p_prosody_ref, + "attn_logprob": attn_logprob, + "attn_soft": attn_soft, + "attn_hard": attn_hard, + "attn_hard_dur": attn_hard_dur, + } + + def forward( + self, + x: torch.Tensor, + speakers: torch.Tensor, + langs: torch.Tensor, + d_control: float = 1.0, + ) -> torch.Tensor: + r"""Forward pass during model inference. + + The forward pass receives phoneme sequence, speaker identities, languages, pitch control and + duration control, conducts a series of operations on these inputs and returns the predicted mel + spectrogram. + + Args: + x (torch.Tensor): Tensor of phoneme sequences. + speakers (torch.Tensor): Tensor of speaker identities. + langs (torch.Tensor): Tensor of language identities. + d_control (float): Duration control parameter. Defaults to 1.0. + + Returns: + torch.Tensor: Predicted mel spectrogram. + """ + # Generate masks for padding positions in the source sequences + src_mask = tools.get_mask_from_lengths( + torch.tensor([x.shape[1]], dtype=torch.int64), + ).to(x.device) + + # Obtain the embeddings for the input + x, embeddings = self.get_embeddings( + token_idx=x, + speaker_idx=speakers, + src_mask=src_mask, + lang_idx=langs, + ) + + # Generate positional encodings + encoding = positional_encoding( + self.emb_dim, + x.shape[1], + ).to(x.device) + + # Process the embeddings through the encoder + x = self.encoder(x, src_mask, embeddings=embeddings, encoding=encoding) + + # Predict prosody at utterance level and phoneme level + u_prosody_pred = self.u_norm( + self.average_utterance_prosody( + u_prosody_pred=self.utterance_prosody_predictor(x=x, mask=src_mask), + src_mask=src_mask, + ), + ) + p_prosody_pred = self.p_norm( + self.phoneme_prosody_predictor( + x=x, + mask=src_mask, + ), + ) + + x = x + self.u_bottle_out(u_prosody_pred) + x = x + self.p_bottle_out(p_prosody_pred) + + x_res = x + + x, _ = self.pitch_adaptor_conv.add_pitch_embedding( + x=x, + mask=src_mask, + ) + + x, _ = self.energy_adaptor.add_energy_embedding( + x=x, + mask=src_mask, + ) + + x, _, embeddings = self.length_regulator.upsample( + x=x, + x_res=x_res, + src_mask=src_mask, + control=d_control, + embeddings=embeddings, + ) + + mel_mask = tools.get_mask_from_lengths( + torch.tensor([x.shape[1]], dtype=torch.int64), + ).to(x.device) + + if x.shape[1] > encoding.shape[1]: + encoding = positional_encoding(self.emb_dim, x.shape[1]).to(x.device) + + decoder_output = self.decoder( + x, + mel_mask, + embeddings=embeddings, + encoding=encoding, + ) + + x = self.to_mel(decoder_output) + x = x.permute((0, 2, 1)) + + return x diff --git a/models/tts/delightful_tts/acoustic_model/acoustic_model2.py b/models/tts/delightful_tts/acoustic_model/acoustic_model2.py new file mode 100644 index 0000000000000000000000000000000000000000..bffb38e9b8e96844d53349e45b5ed9c456baa7e8 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/acoustic_model2.py @@ -0,0 +1,544 @@ +from typing import Dict, Tuple + +import torch +from torch import Tensor, nn +from torch.nn import Module +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from models.config import ( + SUPPORTED_LANGUAGES, + AcousticModelConfigType, + PreprocessingConfig, + symbols, +) +from models.helpers import ( + positional_encoding, + tools, +) +from models.tts.delightful_tts.attention import Conformer +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.reference_encoder import ( + PhonemeLevelProsodyEncoder, + UtteranceLevelProsodyEncoder, +) + +from .alignment_network import AlignmentNetwork +from .duration_adaptor import DurationAdaptor +from .energy_adaptor import EnergyAdaptor +from .phoneme_prosody_predictor import PhonemeProsodyPredictor +from .pitch_adaptor_conv import PitchAdaptorConv + + +class EmbeddingPadded(Module): + r"""EmbeddingPadded is a module that provides embeddings for input indices with support for padding. + + Args: + num_embeddings (int): Size of the dictionary of embeddings. + embedding_dim (int): The size of each embedding vector. + padding_idx (int): The index of the padding token in the input indices. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int): + super().__init__() + padding_mult = torch.ones((num_embeddings, 1), dtype=torch.int64) + padding_mult[padding_idx] = 0 + self.register_buffer("padding_mult", padding_mult) + self.embeddings = nn.parameter.Parameter( + tools.initialize_embeddings((num_embeddings, embedding_dim)), + ) + + def forward(self, idx: Tensor) -> Tensor: + r"""Forward pass of the EmbeddingPadded module. + + Args: + idx (Tensor): Input indices. + + Returns: + Tensor: The embeddings for the input indices. + """ + embeddings_zeroed = self.embeddings * self.padding_mult + x = F.embedding(idx, embeddings_zeroed) + return x + + +class AcousticModel(Module): + r"""The DelightfulTTS AcousticModel class represents a PyTorch module for an acoustic model in text-to-speech (TTS). + The acoustic model is responsible for predicting speech signals from phoneme sequences. + + The model comprises multiple sub-modules including encoder, decoder and various prosody encoders and predictors. + Additionally, a pitch and length adaptor are instantiated. + + Args: + preprocess_config (PreprocessingConfig): Object containing the configuration used for preprocessing the data + model_config (AcousticModelConfigType): Configuration object containing various model parameters + n_speakers (int): Total number of speakers in the dataset + leaky_relu_slope (float, optional): Slope for the leaky relu. Defaults to LEAKY_RELU_SLOPE. + + Note: + For more specific details on the implementation of sub-modules please refer to their individual respective modules. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType, + n_speakers: int, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + self.emb_dim = model_config.encoder.n_hidden + + self.encoder = Conformer( + dim=model_config.encoder.n_hidden, + n_layers=model_config.encoder.n_layers, + n_heads=model_config.encoder.n_heads, + embedding_dim=model_config.speaker_embed_dim + model_config.lang_embed_dim, + p_dropout=model_config.encoder.p_dropout, + kernel_size_conv_mod=model_config.encoder.kernel_size_conv_mod, + with_ff=model_config.encoder.with_ff, + ) + + self.pitch_adaptor_conv = PitchAdaptorConv( + channels_in=model_config.encoder.n_hidden, + channels_hidden=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + emb_kernel_size=model_config.variance_adaptor.emb_kernel_size, + dropout=model_config.variance_adaptor.p_dropout, + leaky_relu_slope=leaky_relu_slope, + ) + + self.energy_adaptor = EnergyAdaptor( + channels_in=model_config.encoder.n_hidden, + channels_hidden=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + emb_kernel_size=model_config.variance_adaptor.emb_kernel_size, + dropout=model_config.variance_adaptor.p_dropout, + leaky_relu_slope=leaky_relu_slope, + ) + + # NOTE: Aligner replaced with AlignmentNetwork + self.aligner = AlignmentNetwork( + in_query_channels=preprocess_config.stft.n_mel_channels, + in_key_channels=model_config.encoder.n_hidden, + attn_channels=preprocess_config.stft.n_mel_channels, + ) + + # NOTE: DurationAdaptor is replacement for LengthAdaptor + self.duration_predictor = DurationAdaptor(model_config) + + self.utterance_prosody_encoder = UtteranceLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.utterance_prosody_predictor = PhonemeProsodyPredictor( + model_config=model_config, + phoneme_level=False, + ) + + self.phoneme_prosody_encoder = PhonemeLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.phoneme_prosody_predictor = PhonemeProsodyPredictor( + model_config=model_config, + phoneme_level=True, + ) + + self.u_bottle_out = nn.Linear( + model_config.reference_encoder.bottleneck_size_u, + model_config.encoder.n_hidden, + ) + + self.u_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_u, + elementwise_affine=False, + ) + + self.p_bottle_out = nn.Linear( + model_config.reference_encoder.bottleneck_size_p, + model_config.encoder.n_hidden, + ) + + self.p_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_p, + elementwise_affine=False, + ) + + self.decoder = Conformer( + dim=model_config.decoder.n_hidden, + n_layers=model_config.decoder.n_layers, + n_heads=model_config.decoder.n_heads, + embedding_dim=model_config.speaker_embed_dim + model_config.lang_embed_dim, + p_dropout=model_config.decoder.p_dropout, + kernel_size_conv_mod=model_config.decoder.kernel_size_conv_mod, + with_ff=model_config.decoder.with_ff, + ) + + self.src_word_emb = EmbeddingPadded( + len( + symbols, + ), # TODO: fix this, check the amount of symbols from the tokenizer + model_config.encoder.n_hidden, + padding_idx=100, # TODO: fix this from training/preprocess/tokenizer_ipa_espeak.py#L59 + ) + # NOTE: here you can manage the speaker embeddings, can be used for the voice export ? + # NOTE: flexibility of the model binded by the n_speaker parameter, maybe I can find another way? + # NOTE: in LIBRITTS there are 2477 speakers, we can add more, just extend the speaker_embed matrix + # Need to think about it more + self.emb_g = nn.Embedding(n_speakers, model_config.speaker_embed_dim) + + self.lang_embed = Parameter( + tools.initialize_embeddings( + (len(SUPPORTED_LANGUAGES), model_config.lang_embed_dim), + ), + ) + + self.to_mel = nn.Linear( + model_config.decoder.n_hidden, + preprocess_config.stft.n_mel_channels, + ) + + def get_embeddings( + self, + token_idx: torch.Tensor, + speaker_idx: torch.Tensor, + src_mask: torch.Tensor, + lang_idx: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Given the tokens, speakers, source mask, and language indices, compute + the embeddings for tokens, speakers and languages and return the + token_embeddings and combined speaker and language embeddings + + Args: + token_idx (torch.Tensor): Tensor of token indices. + speaker_idx (torch.Tensor): Tensor of speaker identities. + src_mask (torch.Tensor): Mask tensor for source sequences. + lang_idx (torch.Tensor): Tensor of language indices. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Token embeddings tensor, + and combined speaker and language embeddings tensor. + """ + # Token embeddings + token_embeddings = self.src_word_emb.forward(token_idx) # [B, T_src, C_hidden] + token_embeddings = token_embeddings.masked_fill(src_mask.unsqueeze(-1), 0.0) + + # NOTE: here you can manage the speaker embeddings, can be used for the voice export ? + speaker_embeds = F.normalize(self.emb_g(speaker_idx)) + + lang_embeds = F.embedding(lang_idx, self.lang_embed) + + # Merge the speaker and language embeddings + embeddings = torch.cat([speaker_embeds, lang_embeds], dim=2) + + # Apply the mask to the embeddings and token embeddings + embeddings = embeddings.masked_fill(src_mask.unsqueeze(-1), 0.0) + token_embeddings = token_embeddings.masked_fill(src_mask.unsqueeze(-1), 0.0) + + return token_embeddings, embeddings + + def average_utterance_prosody( + self, + u_prosody_pred: torch.Tensor, + src_mask: torch.Tensor, + ) -> torch.Tensor: + r"""Compute the average utterance prosody over the length of non-masked elements. + + This method averages the output of the utterance prosody predictor over + the sequence lengths (non-masked elements). This function will return + a tensor with the same first dimension but singleton trailing dimensions. + + Args: + u_prosody_pred (torch.Tensor): Tensor containing the predicted utterance prosody of dimension (batch_size, T, n_features). + src_mask (torch.Tensor): Tensor of dimension (batch_size, T) acting as a mask where masked entries are set to False. + + Returns: + torch.Tensor: Tensor of dimension (batch_size, 1, n_features) containing average utterance prosody over non-masked sequence length. + """ + # Compute the real sequence lengths by negating the mask and summing along the sequence dimension + lengths = ((~src_mask) * 1.0).sum(1) + + # Compute the sum of u_prosody_pred across the sequence length dimension, + # then divide by the sequence lengths tensor to calculate the average. + # This performs a broadcasting operation to account for the third dimension (n_features). + # Return the averaged prosody prediction + return u_prosody_pred.sum(1, keepdim=True) / lengths.view(-1, 1, 1) + + def forward_train( + self, + x: Tensor, + speakers: Tensor, + src_lens: Tensor, + mels: Tensor, + mel_lens: Tensor, + pitches: Tensor, + langs: Tensor, + attn_priors: Tensor, + energies: Tensor, + ) -> Dict[str, Tensor]: + r"""Forward pass during training phase. + + For a given phoneme sequence, speaker identities, sequence lengths, mels, + mel lengths, pitches, language, and attention priors, the forward pass + processes these inputs through the defined architecture. + + Args: + x (Tensor): Tensor of phoneme sequence. + speakers (Tensor): Tensor of speaker identities. + src_lens (Tensor): Long tensor representing the lengths of source sequences. + mels (Tensor): Tensor of mel spectrograms. + mel_lens (Tensor): Long tensor representing the lengths of mel sequences. + pitches (Tensor): Tensor of pitch values. + langs (Tensor): Tensor of language identities. + attn_priors (Tensor): Prior attention values. + energies (Tensor): Tensor of energy values. + + Returns: + Dict[str, Tensor]: Returns the prediction outputs as a dictionary. + """ + # Generate masks for padding positions in the source sequences and mel sequences + src_mask = tools.get_mask_from_lengths(src_lens) + mel_mask = tools.get_mask_from_lengths(mel_lens) + + token_embeddings, embeddings = self.get_embeddings( + token_idx=x, + speaker_idx=speakers, + src_mask=src_mask, + lang_idx=langs, + ) + token_embeddings = token_embeddings.to(src_mask.device) + embeddings = embeddings.to(src_mask.device) + + encoding = positional_encoding( + self.emb_dim, + max(x.shape[1], int(mel_lens.max().item())), + ).to(src_mask.device) + + attn_logprob, attn_soft, attn_hard, attn_hard_dur = self.aligner.forward( + x=token_embeddings, + y=mels.transpose(1, 2), + x_mask=~src_mask[:, None], + y_mask=~mel_mask[:, None], + attn_priors=attn_priors, + ) + attn_hard_dur = attn_hard_dur.to(src_mask.device) + + x = self.encoder( + token_embeddings, + src_mask, + embeddings=embeddings, + encoding=encoding, + ) + + u_prosody_ref = self.u_norm( + self.utterance_prosody_encoder(mels=mels, mel_lens=mel_lens), + ) + u_prosody_pred = self.u_norm( + self.average_utterance_prosody( + u_prosody_pred=self.utterance_prosody_predictor(x=x, mask=src_mask), + src_mask=src_mask, + ), + ) + + p_prosody_ref = self.p_norm( + self.phoneme_prosody_encoder( + x=x, + src_mask=src_mask, + mels=mels, + mel_lens=mel_lens, + encoding=encoding, + ), + ) + p_prosody_pred = self.p_norm( + self.phoneme_prosody_predictor( + x=x, + mask=src_mask, + ), + ) + + x = x + self.u_bottle_out(u_prosody_pred) + x = x + self.p_bottle_out(p_prosody_pred) + + # Save the residual for later use + x_res = x + + x, pitch_prediction, avg_pitch_target = ( + self.pitch_adaptor_conv.add_pitch_embedding_train( + x=x, + target=pitches, + dr=attn_hard_dur, + mask=src_mask, + ) + ) + + energies = energies.to(src_mask.device) + + x, energy_pred, avg_energy_target = ( + self.energy_adaptor.add_energy_embedding_train( + x=x, + target=energies, + dr=attn_hard_dur, + mask=src_mask, + ) + ) + + ( + alignments_duration_pred, + log_duration_prediction, + x, + alignments, + ) = self.duration_predictor.forward_train( + encoder_output=x, + encoder_output_res=x_res, + duration_target=attn_hard_dur, + src_mask=src_mask, + mel_lens=mel_lens, + ) + + # Change the embedding shape to match the decoder output + embeddings_out = embeddings.repeat( + 1, + encoding.shape[1] // embeddings.shape[1] + 1, + 1, + )[:, : encoding.shape[1], :] + + # Decode the encoder output to pred mel spectrogram + decoder_output = self.decoder.forward( + x.transpose(1, 2), + mel_mask, + embeddings=embeddings_out, + encoding=encoding, + ) + + y_pred: torch.Tensor = self.to_mel(decoder_output) + y_pred = y_pred.permute((0, 2, 1)) + + return { + "y_pred": y_pred, + "pitch_prediction": pitch_prediction, + "pitch_target": avg_pitch_target, + "energy_pred": energy_pred, + "energy_target": avg_energy_target, + "log_duration_prediction": log_duration_prediction, + "u_prosody_pred": u_prosody_pred, + "u_prosody_ref": u_prosody_ref, + "p_prosody_pred": p_prosody_pred, + "p_prosody_ref": p_prosody_ref, + "alignments": alignments, + "alignments_duration_pred": alignments_duration_pred, + "attn_logprob": attn_logprob, + "attn_soft": attn_soft, + "attn_hard": attn_hard, + "attn_hard_dur": attn_hard_dur, + } + + def forward( + self, + x: torch.Tensor, + speakers: torch.Tensor, + langs: torch.Tensor, + d_control: float = 1.0, + ) -> torch.Tensor: + r"""Forward pass during model inference. + + The forward pass receives phoneme sequence, speaker identities, languages, pitch control and + duration control, conducts a series of operations on these inputs and returns the predicted mel + spectrogram. + + Args: + x (torch.Tensor): Tensor of phoneme sequences. + speakers (torch.Tensor): Tensor of speaker identities. + langs (torch.Tensor): Tensor of language identities. + d_control (float): Duration control parameter. Defaults to 1.0. + + Returns: + torch.Tensor: Predicted mel spectrogram. + """ + # Generate masks for padding positions in the source sequences + src_mask = tools.get_mask_from_lengths( + torch.tensor([x.shape[1]], dtype=torch.int64), + ).to(x.device) + + # Obtain the embeddings for the input + x, embeddings = self.get_embeddings( + token_idx=x, + speaker_idx=speakers, + src_mask=src_mask, + lang_idx=langs, + ) + + # Generate positional encodings + encoding = positional_encoding( + self.emb_dim, + x.shape[1], + ).to(x.device) + + # Process the embeddings through the encoder + x = self.encoder(x, src_mask, embeddings=embeddings, encoding=encoding) + + # Predict prosody at utterance level and phoneme level + u_prosody_pred = self.u_norm( + self.average_utterance_prosody( + u_prosody_pred=self.utterance_prosody_predictor(x=x, mask=src_mask), + src_mask=src_mask, + ), + ) + p_prosody_pred = self.p_norm( + self.phoneme_prosody_predictor( + x=x, + mask=src_mask, + ), + ) + + x = x + self.u_bottle_out(u_prosody_pred) + x = x + self.p_bottle_out(p_prosody_pred) + + x, _ = self.pitch_adaptor_conv.add_pitch_embedding( + x=x, + mask=src_mask, + ) + + x, _ = self.energy_adaptor.add_energy_embedding( + x=x, + mask=src_mask, + ) + + _, x, _, _ = self.duration_predictor.forward( + encoder_output=x, + src_mask=src_mask, + d_control=d_control, + ) + + mel_mask = tools.get_mask_from_lengths( + torch.tensor( + [x.shape[2]], + dtype=torch.int64, + ), + ).to(x.device) + + if x.shape[1] > encoding.shape[1]: + encoding = positional_encoding(self.emb_dim, x.shape[2]).to(x.device) + + # Change the embedding shape to match the decoder output + embeddings_out = embeddings.repeat( + 1, + mel_mask.shape[1] // embeddings.shape[1] + 1, + 1, + )[:, : mel_mask.shape[1], :] + + decoder_output = self.decoder.forward( + x.transpose(1, 2), + mel_mask, + embeddings=embeddings_out, + encoding=encoding, + ) + + x = self.to_mel(decoder_output) + x = x.permute((0, 2, 1)) + + return x diff --git a/models/tts/delightful_tts/acoustic_model/aligner.py b/models/tts/delightful_tts/acoustic_model/aligner.py new file mode 100644 index 0000000000000000000000000000000000000000..e62aba38826f0a4fdf00b5831e68262e756cbb8f --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/aligner.py @@ -0,0 +1,178 @@ +from typing import Tuple + +import torch +from torch import nn +from torch.nn import Module + +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE + +from .mas import b_mas + + +class Aligner(Module): + r"""DEPRECATED: Aligner class represents a PyTorch module responsible for alignment tasks + in a sequence-to-sequence model. It uses convolutional layers combined with + LeakyReLU activation functions to project inputs to a hidden representation. + The class utilizes both softmax and log-softmax to calculate softmax + along dimension 3. + + Args: + d_enc_in (int): Number of channels in the input for the encoder. + d_dec_in (int): Number of channels in the input for the decoder. + d_hidden (int): Number of channels in the output (hidden layers). + kernel_size_enc (int, optional): Size of the convolving kernel for encoder, default is 3. + kernel_size_dec (int, optional): Size of the convolving kernel for decoder, default is 7. + temperature (float, optional): The temperature value applied in Gaussian isotropic + attention mechanism, default is 0.0005. + leaky_relu_slope (float, optional): Controls the angle of the negative slope of + LeakyReLU activation, default is LEAKY_RELU_SLOPE. + + """ + + def __init__( + self, + d_enc_in: int, + d_dec_in: int, + d_hidden: int, + kernel_size_enc: int = 3, + kernel_size_dec: int = 7, + temperature: float = 0.0005, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + self.temperature = temperature + + self.softmax = torch.nn.Softmax(dim=3) + self.log_softmax = torch.nn.LogSoftmax(dim=3) + + self.key_proj = nn.Sequential( + nn.Conv1d( + d_enc_in, + d_hidden, + kernel_size=kernel_size_enc, + padding=kernel_size_enc // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.Conv1d( + d_hidden, + d_hidden, + kernel_size=kernel_size_enc, + padding=kernel_size_enc // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + ) + + self.query_proj = nn.Sequential( + nn.Conv1d( + d_dec_in, + d_hidden, + kernel_size=kernel_size_dec, + padding=kernel_size_dec // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.Conv1d( + d_hidden, + d_hidden, + kernel_size=kernel_size_dec, + padding=kernel_size_dec // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.Conv1d( + d_hidden, + d_hidden, + kernel_size=kernel_size_dec, + padding=kernel_size_dec // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + ) + + def binarize_attention_parallel( + self, + attn: torch.Tensor, + in_lens: torch.Tensor, + out_lens: torch.Tensor, + ) -> torch.Tensor: + r"""For training purposes only! Binarizes attention with MAS. + Binarizes the attention tensor using Maximum Attention Strategy (MAS). + + This process is applied for training purposes only and the resulting + binarized attention tensor will no longer receive a gradient in the + backpropagation process. + + Args: + attn (Tensor): The attention tensor. Must be of shape (B, 1, max_mel_len, max_text_len), + where B represents the batch size, max_mel_len represents the maximum length + of the mel spectrogram, and max_text_len represents the maximum length of the text. + in_lens (Tensor): A 1D tensor of shape (B,) that contains the input sequence lengths, + which likely corresponds to text sequence lengths. + out_lens (Tensor): A 1D tensor of shape (B,) that contains the output sequence lengths, + which likely corresponds to mel spectrogram lengths. + + Returns: + Tensor: The binarized attention tensor. The output tensor has the same shape as the input `attn` tensor. + """ + with torch.no_grad(): + attn_cpu = attn.data.cpu().numpy() + attn_out = b_mas( + attn_cpu, + in_lens.cpu().numpy(), + out_lens.cpu().numpy(), + width=1, + ) + return torch.from_numpy(attn_out) + + def forward( + self, + enc_in: torch.Tensor, + dec_in: torch.Tensor, + enc_len: torch.Tensor, + dec_len: torch.Tensor, + enc_mask: torch.Tensor, + attn_prior: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Performs the forward pass through the Aligner module. + + Args: + enc_in (Tensor): The text encoder outputs. + Must be of shape (B, C_1, T_1), where B is the batch size, C_1 the number of + channels in encoder inputs, + and T_1 the sequence length of encoder inputs. + dec_in (Tensor): The data to align with encoder outputs. + Must be of shape (B, C_2, T_2), where C_2 is the number of channels in decoder inputs, + and T_2 the sequence length of decoder inputs. + enc_len (Tensor): 1D tensor representing the lengths of each sequence in the batch in `enc_in`. + dec_len (Tensor): 1D tensor representing the lengths of each sequence in the batch in `dec_in`. + enc_mask (Tensor): Binary mask tensor used to avoid attention to certain timesteps. + attn_prior (Tensor): Previous attention values for attention calculation. + + Returns: + Tuple[Tensor, Tensor, Tensor, Tensor]: Returns a tuple of Tensors representing the log-probability, soft attention, hard attention, and hard attention duration. + """ + queries = dec_in.float() + keys = enc_in.float() + keys_enc = self.key_proj(keys) # B x n_attn_dims x T2 + queries_enc = self.query_proj(queries) + + # Simplistic Gaussian Isotopic Attention + attn = ( + queries_enc[:, :, :, None] - keys_enc[:, :, None] + ) ** 2 # B x n_attn_dims x T1 x T2 + attn = -self.temperature * attn.sum(1, keepdim=True) + + if attn_prior is not None: + # print(f"AlignmentEncoder \t| mel: {queries.shape} phone: {keys.shape} + # mask: {mask.shape} attn: {attn.shape} attn_prior: {attn_prior.shape}") + attn = self.log_softmax(attn) + torch.log( + attn_prior.permute((0, 2, 1))[:, None] + 1e-8, + ) + # print(f"AlignmentEncoder \t| After prior sum attn: {attn.shape}")""" + + attn_logprob = attn.clone() + + if enc_mask is not None: + attn.masked_fill(enc_mask.unsqueeze(1).unsqueeze(1), -float("inf")) + + attn_soft = self.softmax(attn) # softmax along T2 + attn_hard = self.binarize_attention_parallel(attn_soft, enc_len, dec_len) + attn_hard_dur = attn_hard.sum(2)[:, 0, :] + return attn_logprob, attn_soft, attn_hard, attn_hard_dur diff --git a/models/tts/delightful_tts/acoustic_model/alignment_network.py b/models/tts/delightful_tts/acoustic_model/alignment_network.py new file mode 100644 index 0000000000000000000000000000000000000000..f05f330bc197337f5550243141a50216e1475c0c --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/alignment_network.py @@ -0,0 +1,253 @@ +from typing import Optional, Tuple + +import numpy as np +import torch +from torch import Tensor, nn + + +def maximum_path( + value: Tensor, + mask: Tensor, + max_neg_val: Optional[float] = None, +): + """Monotonic alignment search algorithm + Numpy-friendly version. It's about 4 times faster than torch version. + value: [b, t_x, t_y] + mask: [b, t_x, t_y] + """ + if max_neg_val is None: + max_neg_val = -np.inf # Patch for Sphinx complaint + value = value * mask + + device = value.device + dtype = value.dtype + value = value.cpu().detach().numpy() + mask = mask.cpu().detach().numpy().astype(bool) + + b, t_x, t_y = value.shape + direction = np.zeros(value.shape, dtype=np.int64) + v = np.zeros((b, t_x), dtype=np.float32) + x_range = np.arange(t_x, dtype=np.float32).reshape(1, -1) + for j in range(t_y): + v0 = np.pad(v, [[0, 0], [1, 0]], mode="constant", constant_values=max_neg_val)[ + :, + :-1, + ] + v1 = v + max_mask = v1 >= v0 + v_max = np.where(max_mask, v1, v0) + direction[:, :, j] = max_mask + + index_mask = x_range <= j + v = np.where(index_mask, v_max + value[:, :, j], max_neg_val) + direction = np.where(mask, direction, 1) + + path = np.zeros(value.shape, dtype=np.float32) + index = mask[:, :, 0].sum(1).astype(np.int64) - 1 # type: ignore + index_range = np.arange(b) + for j in reversed(range(t_y)): + path[index_range, index, j] = 1 + index = index + direction[index_range, index, j] - 1 + path = path * mask.astype(np.float32) # type: ignore + path = torch.from_numpy(path).to(device=device, dtype=dtype) + return path + + +class AlignmentNetwork(torch.nn.Module): + r"""Aligner Network for learning alignment between the input text and the model output with Gaussian Attention. + + The architecture is as follows: + query -> conv1d -> relu -> conv1d -> relu -> conv1d -> L2_dist -> softmax -> alignment + key -> conv1d -> relu -> conv1d -----------------------^ + + Args: + in_query_channels (int): Number of channels in the query network. + in_key_channels (int): Number of channels in the key network. + attn_channels (int): Number of inner channels in the attention layers. + temperature (float, optional): Temperature for the softmax. Defaults to 0.0005. + """ + + def __init__( + self, + in_query_channels: int, + in_key_channels: int, + attn_channels: int, + temperature: float = 0.0005, + ): + super().__init__() + self.temperature = temperature + self.softmax = torch.nn.Softmax(dim=3) + self.log_softmax = torch.nn.LogSoftmax(dim=3) + + self.key_layer = nn.Sequential( + nn.Conv1d( + in_key_channels, + in_key_channels * 2, + kernel_size=3, + padding=1, + bias=True, + ), + torch.nn.ReLU(), + nn.Conv1d( + in_key_channels * 2, + attn_channels, + kernel_size=1, + padding=0, + bias=True, + ), + ) + + self.query_layer = nn.Sequential( + nn.Conv1d( + in_query_channels, + in_query_channels * 2, + kernel_size=3, + padding=1, + bias=True, + ), + torch.nn.ReLU(), + nn.Conv1d( + in_query_channels * 2, + in_query_channels, + kernel_size=1, + padding=0, + bias=True, + ), + torch.nn.ReLU(), + nn.Conv1d( + in_query_channels, + attn_channels, + kernel_size=1, + padding=0, + bias=True, + ), + ) + + self.init_layers() + + def init_layers(self): + r"""Initialize the weights of the key and query layers using Xavier uniform initialization. + + The gain is calculated based on the activation function: ReLU for the first layer and linear for the rest. + """ + torch.nn.init.xavier_uniform_( + self.key_layer[0].weight, + gain=torch.nn.init.calculate_gain("relu"), + ) + torch.nn.init.xavier_uniform_( + self.key_layer[2].weight, + gain=torch.nn.init.calculate_gain("linear"), + ) + torch.nn.init.xavier_uniform_( + self.query_layer[0].weight, + gain=torch.nn.init.calculate_gain("relu"), + ) + torch.nn.init.xavier_uniform_( + self.query_layer[2].weight, + gain=torch.nn.init.calculate_gain("linear"), + ) + torch.nn.init.xavier_uniform_( + self.query_layer[4].weight, + gain=torch.nn.init.calculate_gain("linear"), + ) + + def _forward_aligner( + self, + queries: Tensor, + keys: Tensor, + mask: Optional[Tensor] = None, + attn_prior: Optional[Tensor] = None, + ) -> Tuple[Tensor, Tensor]: + r"""Forward pass of the aligner encoder. + + Args: + queries (Tensor): Input queries of shape [B, C, T_de]. + keys (Tensor): Input keys of shape [B, C_emb, T_en]. + mask (Optional[Tensor], optional): Mask of shape [B, T_de]. Defaults to None. + attn_prior (Optional[Tensor], optional): Prior attention tensor. Defaults to None. + + Returns: + Tuple[Tensor, Tensor]: A tuple containing the soft attention mask of shape [B, 1, T_en, T_de] and + log probabilities of shape [B, 1, T_en , T_de]. + """ + key_out = self.key_layer(keys) + query_out = self.query_layer(queries) + attn_factor = (query_out[:, :, :, None] - key_out[:, :, None]) ** 2 + attn_logp = -self.temperature * attn_factor.sum(1, keepdim=True) + if attn_prior is not None: + attn_logp = self.log_softmax(attn_logp) + torch.log( + attn_prior[:, None] + 1e-8, + ).permute((0, 1, 3, 2)) + + if mask is not None: + attn_logp.data.masked_fill_(~mask.bool().unsqueeze(2), -float("inf")) + + attn = self.softmax(attn_logp) + return attn, attn_logp + + def forward( + self, + x: Tensor, + y: Tensor, + x_mask: Tensor, + y_mask: Tensor, + attn_priors: Tensor, + ) -> Tuple[ + Tensor, + Tensor, + Tensor, + Tensor, + ]: + r"""Aligner forward pass. + + 1. Compute a mask to apply to the attention map. + 2. Run the alignment network. + 3. Apply MAS to compute the hard alignment map. + 4. Compute the durations from the hard alignment map. + + Args: + x (torch.Tensor): Input sequence. + y (torch.Tensor): Output sequence. + x_mask (torch.Tensor): Input sequence mask. + y_mask (torch.Tensor): Output sequence mask. + attn_priors (torch.Tensor): Prior for the aligner network map. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + Durations from the hard alignment map, soft alignment potentials, log scale alignment potentials, + hard alignment map. + + Shapes: + - x: :math:`[B, T_en, C_en]` + - y: :math:`[B, T_de, C_de]` + - x_mask: :math:`[B, 1, T_en]` + - y_mask: :math:`[B, 1, T_de]` + - attn_priors: :math:`[B, T_de, T_en]` + + - aligner_durations: :math:`[B, T_en]` + - aligner_soft: :math:`[B, T_de, T_en]` + - aligner_logprob: :math:`[B, 1, T_de, T_en]` + - aligner_mas: :math:`[B, T_de, T_en]` + """ + # [B, 1, T_en, T_de] + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + + aligner_soft, aligner_logprob = self._forward_aligner( + y.transpose(1, 2), + x.transpose(1, 2), + x_mask, + attn_priors, + ) + + aligner_mas = maximum_path( + aligner_soft.squeeze(1).transpose(1, 2).contiguous(), + attn_mask.squeeze(1).contiguous(), + ) + aligner_durations = torch.sum(aligner_mas, -1).int() + + # [B, T_max2, T_max] + aligner_soft = aligner_soft.squeeze(1) + # [B, T_max, T_max2] -> [B, T_max2, T_max] + aligner_mas = aligner_mas.transpose(1, 2) + + return aligner_logprob, aligner_soft, aligner_mas, aligner_durations diff --git a/models/tts/delightful_tts/acoustic_model/duration_adaptor.py b/models/tts/delightful_tts/acoustic_model/duration_adaptor.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb6d298654db3a7d544f1ce30d890841898c6bc --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/duration_adaptor.py @@ -0,0 +1,252 @@ +from typing import List, Optional, Tuple + +import torch +from torch import Tensor +from torch.nn import Module +import torch.nn.functional as F + +from models.config import AcousticModelConfigType + +from .variance_predictor import VariancePredictor + + +class DurationAdaptor(Module): + """DurationAdaptor is a module that adapts the duration of the input sequence. + + Args: + model_config (AcousticModelConfigType): Configuration object containing model parameters. + """ + + def __init__( + self, + model_config: AcousticModelConfigType, + ): + super().__init__() + # Initialize the duration predictor + self.duration_predictor = VariancePredictor( + channels_in=model_config.encoder.n_hidden, + channels=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + p_dropout=model_config.variance_adaptor.p_dropout, + ) + + @staticmethod + def convert_pad_shape(pad_shape: List[List[int]]) -> List[int]: + r"""Convert the padding shape from a list of lists to a flat list. + + Args: + pad_shape (List[List[int]]): Padding shape as a list of lists. + + Returns: + List[int]: Padding shape as a flat list. + """ + pad_list = pad_shape[::-1] + return [item for sublist in pad_list for item in sublist] + + @staticmethod + def generate_path(duration: Tensor, mask: Tensor) -> Tensor: + r"""Generate a path based on the duration and mask. + + Args: + duration (Tensor): Duration tensor. + mask (Tensor): Mask tensor. + + Returns: + Tensor: Path tensor. + + Shapes: + - duration: :math:`[B, T_en]` + - mask: :math:'[B, T_en, T_de]` + - path: :math:`[B, T_en, T_de]` + """ + b, t_x, t_y = mask.shape + cum_duration = torch.cumsum(duration, 1) + + cum_duration_flat = cum_duration.view(b * t_x) + path = DurationAdaptor.sequence_mask(cum_duration_flat, t_y).to(mask.dtype) + path = path.view(b, t_x, t_y) + pad_shape = DurationAdaptor.convert_pad_shape([[0, 0], [1, 0], [0, 0]]) + path = path - F.pad(path, pad_shape)[:, :-1] + path = path * mask + return path + + # from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1 + @staticmethod + def sequence_mask(sequence_length: Tensor, max_len: Optional[int] = None) -> Tensor: + """Create a sequence mask for filtering padding in a sequence tensor. + + Args: + sequence_length (torch.Tensor): Sequence lengths. + max_len (int, Optional): Maximum sequence length. Defaults to None. + + Returns: + torch.Tensor: Sequence mask. + + Shapes: + - mask: :math:`[B, T_max]` + """ + if max_len is None: + max_len = int(sequence_length.max()) + + seq_range = torch.arange( + max_len, + dtype=sequence_length.dtype, + device=sequence_length.device, + ) + # B x T_max + return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1) + + @staticmethod + def generate_attn( + dr: Tensor, + x_mask: Tensor, + y_mask: Optional[Tensor] = None, + ) -> Tensor: + """Generate an attention mask from the linear scale durations. + + Args: + dr (Tensor): Linear scale durations. + x_mask (Tensor): Mask for the input (character) sequence. + y_mask (Tensor): Mask for the output (spectrogram) sequence. Compute it from the predicted durations + if None. Defaults to None. + + Shapes + - dr: :math:`(B, T_{en})` + - x_mask: :math:`(B, T_{en})` + - y_mask: :math:`(B, T_{de})` + """ + # compute decode mask from the durations + if y_mask is None: + y_lengths = dr.sum(1).long() + y_lengths[y_lengths < 1] = 1 + sequence_mask = DurationAdaptor.sequence_mask(y_lengths, None) + y_mask = torch.unsqueeze(sequence_mask, 1).to(dr.dtype) + + # compute the attention mask + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + attn = DurationAdaptor.generate_path(dr, attn_mask.squeeze(1)).to(dr.dtype) + return attn + + def _expand_encoder_with_durations( + self, + encoder_output: Tensor, + duration_target: Tensor, + x_mask: Tensor, + mel_lens: Tensor, + ) -> Tuple[Tensor, Tensor, Tensor]: + r"""Expand the encoder output with durations. + + Args: + encoder_output (Tensor): Encoder output. + duration_target (Tensor): Target durations. + x_mask (Tensor): Mask for the input sequence. + mel_lens (Tensor): Lengths of the mel spectrograms. + + Returns: + Tuple[Tensor, Tensor, Tensor]: Tuple containing the mask for the output sequence, the attention mask, and the expanded encoder output. + """ + y_mask = torch.unsqueeze( + DurationAdaptor.sequence_mask(mel_lens, None), + 1, + ).to(encoder_output.dtype) + attn = self.generate_attn(duration_target, x_mask, y_mask) + + encoder_output_ex = torch.einsum( + "kmn, kmj -> kjn", + [attn.float(), encoder_output], + ) + + return y_mask, attn, encoder_output_ex + + def forward_train( + self, + encoder_output: Tensor, + encoder_output_res: Tensor, + duration_target: Tensor, + src_mask: Tensor, + mel_lens: Tensor, + ): + r"""Forward pass of the DurationAdaptor during training. + + Args: + encoder_output (Tensor): Encoder output. + encoder_output_res (Tensor): Encoder output. + duration_target (Tensor): Target durations. + src_mask (Tensor): Source mask. + mel_lens (Tensor): Lengths of the mel spectrograms. + + Returns: + Tuple: Tuple containing the predicted alignments, log durations, mask for the output sequence, expanded encoder output, and the transposed attention mask. + """ + log_duration_pred = self.duration_predictor.forward( + x=encoder_output_res.detach(), + mask=src_mask, + ) # [B, C_hidden, T_src] -> [B, T_src] + + y_mask, attn, encoder_output_dr = self._expand_encoder_with_durations( + encoder_output, + duration_target, + x_mask=~src_mask[:, None], + mel_lens=mel_lens, + ) + + duration_target = torch.log(duration_target + 1) + duration_pred = torch.exp(log_duration_pred) - 1 + + alignments_duration_pred = self.generate_attn( + duration_pred, + src_mask.unsqueeze(1), + y_mask, + ) # [B, T_max, T_max2'] + + return ( + alignments_duration_pred, + log_duration_pred, + encoder_output_dr, + attn.transpose(1, 2), + ) + + def forward(self, encoder_output: Tensor, src_mask: Tensor, d_control: float = 1.0): + r"""Forward pass of the DurationAdaptor. + + Args: + encoder_output (Tensor): Encoder output. + src_mask (Tensor): Source mask. + d_control (float): Duration control. Defaults to 1.0. + + Returns: + Tuple: Tuple containing the expanded encoder output, log durations, predicted durations, mask for the output sequence, and the attention mask. + """ + log_duration_pred = self.duration_predictor( + x=encoder_output.detach(), + mask=src_mask, + ) # [B, C_hidden, T_src] -> [B, T_src] + + duration_pred = ( + (torch.exp(log_duration_pred) - 1) * (~src_mask) * d_control + ) # -> [B, T_src] + + # duration_pred[duration_pred < 1] = 1.0 # -> [B, T_src] + duration_pred = torch.where( + duration_pred < 1, + torch.ones_like(duration_pred), + duration_pred, + ) # -> [B, T_src] + + duration_pred = torch.round(duration_pred) # -> [B, T_src] + mel_lens = duration_pred.sum(1) # -> [B,] + + _, attn, encoder_output_dr = self._expand_encoder_with_durations( + encoder_output, + duration_pred.squeeze(1), + ~src_mask[:, None], + mel_lens, + ) + + return ( + log_duration_pred, + encoder_output_dr, + duration_pred, + attn.transpose(1, 2), + ) diff --git a/models/tts/delightful_tts/acoustic_model/embedding.py b/models/tts/delightful_tts/acoustic_model/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..4183d8d7db7bd686d691e347d84dac810d459a59 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/embedding.py @@ -0,0 +1,36 @@ +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F + + +class Embedding(Module): + r"""Class represents a simple embedding layer but without any learning of the embeddings. + The embeddings are initialized with random values and kept static throughout training (They are parameters, not model's state). + + Args: + num_embeddings (int): Size of the dictionary of embeddings, typically size of the vocabulary. + embedding_dim (int): The size of each embedding vector. + + Returns: + torch.Tensor: An output tensor resulting from the lookup operation. + """ + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + ): + super().__init__() + self.embeddings = nn.Parameter(torch.randn(num_embeddings, embedding_dim)) + + def forward(self, idx: torch.Tensor) -> torch.Tensor: + r"""Forward propagation for the Embedding implementation. + + Args: + idx (torch.Tensor): A tensor containing the indices of the embeddings to be accessed. + + Returns: + torch.Tensor: An output tensor resulting from the lookup operation. + """ + return F.embedding(idx, self.embeddings) diff --git a/models/tts/delightful_tts/acoustic_model/energy_adaptor.py b/models/tts/delightful_tts/acoustic_model/energy_adaptor.py new file mode 100644 index 0000000000000000000000000000000000000000..157f2a7d51e049d365e149b7bfd310f5e88385dc --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/energy_adaptor.py @@ -0,0 +1,178 @@ +from typing import Tuple + +import torch +from torch import nn + +from .helpers import average_over_durations +from .variance_predictor import VariancePredictor + + +class EnergyAdaptor(nn.Module): + """Variance Adaptor with an added 1D conv layer. Used to + get energy embeddings. + + Args: + channels_in (int): Number of in channels for conv layers. + channels_out (int): Number of out channels. + kernel_size (int): Size the kernel for the conv layers. + dropout (float): Probability of dropout. + leaky_relu_slope (float): Slope for the leaky relu. + emb_kernel_size (int): Size the kernel for the pitch embedding. + + Inputs: inputs, mask + - **inputs** (batch, time1, dim): Tensor containing input vector + - **target** (batch, 1, time2): Tensor containing the energy target + - **dr** (batch, time1): Tensor containing aligner durations vector + - **mask** (batch, time1): Tensor containing indices to be masked + Returns: + - **energy prediction** (batch, 1, time1): Tensor produced by energy predictor + - **energy embedding** (batch, channels, time1): Tensor produced energy adaptor + - **average energy target(train only)** (batch, 1, time1): Tensor produced after averaging over durations + + """ + + def __init__( + self, + channels_in: int, + channels_hidden: int, + channels_out: int, + kernel_size: int, + dropout: float, + leaky_relu_slope: float, + emb_kernel_size: int, + ): + super().__init__() + self.energy_predictor = VariancePredictor( + channels_in=channels_in, + channels=channels_hidden, + channels_out=channels_out, + kernel_size=kernel_size, + p_dropout=dropout, + leaky_relu_slope=leaky_relu_slope, + ) + self.energy_emb = nn.Conv1d( + 1, + channels_hidden, + kernel_size=emb_kernel_size, + padding=int((emb_kernel_size - 1) / 2), + ) + + def get_energy_embedding_train( + self, + x: torch.Tensor, + target: torch.Tensor, + dr: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Function is used during training to get the energy prediction, average energy target, and energy embedding. + + Args: + x (torch.Tensor): A 3D tensor of shape [B, T_src, C] where B is the batch size, + T_src is the source sequence length, and C is the number of channels. + target (torch.Tensor): A 3D tensor of shape [B, 1, T_max2] where B is the batch size, + T_max2 is the maximum target sequence length. + dr (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the durations. + mask (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the mask. + + Returns: + energy_pred (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the energy prediction. + avg_energy_target (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the average energy target. + energy_emb (torch.Tensor): A 3D tensor of shape [B, C, T_src] where B is the batch size, + C is the number of channels, T_src is the source sequence length. The values represent the energy embedding. + Shapes: + x: :math: `[B, T_src, C]` + target: :math: `[B, 1, T_max2]` + dr: :math: `[B, T_src]` + mask: :math: `[B, T_src]` + """ + energy_pred = self.energy_predictor.forward(x, mask) + energy_pred = energy_pred.unsqueeze(1) + + avg_energy_target = average_over_durations(target, dr) + energy_emb = self.energy_emb(avg_energy_target) + + return energy_pred, avg_energy_target, energy_emb + + def add_energy_embedding_train( + self, + x: torch.Tensor, + target: torch.Tensor, + dr: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Add energy embedding during training. + + This method calculates the energy embedding and adds it to the input tensor 'x'. + It also returns the predicted energy and the average target energy. + + Args: + x (torch.Tensor): The input tensor to which the energy embedding will be added. + target (torch.Tensor): The target tensor used in the energy embedding calculation. + dr (torch.Tensor): The duration tensor used in the energy embedding calculation. + mask (torch.Tensor): The mask tensor used in the energy embedding calculation. + + Returns: + x (torch.Tensor): The input tensor with added energy embedding. + energy_pred (torch.Tensor): The predicted energy tensor. + avg_energy_target (torch.Tensor): The average target energy tensor. + """ + energy_pred, avg_energy_target, energy_emb = self.get_energy_embedding_train( + x=x, + target=target, + dr=dr, + mask=mask, + ) + x_energy = x + energy_emb.transpose(1, 2) + return x_energy, energy_pred, avg_energy_target + + def get_energy_embedding( + self, + x: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Function is used during inference to get the energy embedding and energy prediction. + + Args: + x (torch.Tensor): A 3D tensor of shape [B, T_src, C] where B is the batch size, + T_src is the source sequence length, and C is the number of channels. + mask (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the mask. + + Returns: + energy_emb_pred (torch.Tensor): A 3D tensor of shape [B, C, T_src] where B is the batch size, + C is the number of channels, T_src is the source sequence length. The values represent the energy embedding. + energy_pred (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the energy prediction. + """ + energy_pred = self.energy_predictor.forward(x, mask) + energy_pred = energy_pred.unsqueeze(1) + + energy_emb_pred = self.energy_emb(energy_pred) + return energy_emb_pred, energy_pred + + def add_energy_embedding( + self, + x: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Add energy embedding during inference. + + This method calculates the energy embedding and adds it to the input tensor 'x'. + It also returns the predicted energy. + + Args: + x (torch.Tensor): The input tensor to which the energy embedding will be added. + mask (torch.Tensor): The mask tensor used in the energy embedding calculation. + energy_transform (Callable): A function to transform the energy prediction. + + Returns: + x (torch.Tensor): The input tensor with added energy embedding. + energy_pred (torch.Tensor): The predicted energy tensor. + """ + energy_emb_pred, energy_pred = self.get_energy_embedding(x, mask) + x_energy = x + energy_emb_pred.transpose(1, 2) + return x_energy, energy_pred diff --git a/models/tts/delightful_tts/acoustic_model/helpers.py b/models/tts/delightful_tts/acoustic_model/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..fadd8087d0ce8ac8d29f03ab21896c3bcd753f3f --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/helpers.py @@ -0,0 +1,42 @@ +import torch + + +def average_over_durations(values: torch.Tensor, durs: torch.Tensor) -> torch.Tensor: + r"""Function calculates the average of values over specified durations. + + Args: + values (torch.Tensor): A 3D tensor of shape [B, 1, T_de] where B is the batch size, + T_de is the duration of each element in the batch. The values + represent some quantity that needs to be averaged over durations. + durs (torch.Tensor): A 2D tensor of shape [B, T_en] where B is the batch size, + T_en is the number of elements in each batch. The values represent + the durations over which the averaging needs to be done. + + Returns: + avg (torch.Tensor): A 3D tensor of shape [B, 1, T_en] where B is the batch size, + T_en is the number of elements in each batch. The values represent + the average of the input values over the specified durations. + + Note: + The function uses PyTorch operations for efficient computation on GPU. + + Shapes: + - values: :math:`[B, 1, T_de]` + - durs: :math:`[B, T_en]` + - avg: :math:`[B, 1, T_en]` + """ + durs_cums_ends = torch.cumsum(durs, dim=1).long() + durs_cums_starts = torch.nn.functional.pad(durs_cums_ends[:, :-1], (1, 0)) + values_nonzero_cums = torch.nn.functional.pad(torch.cumsum(values != 0.0, dim=2), (1, 0)) + values_cums = torch.nn.functional.pad(torch.cumsum(values, dim=2), (1, 0)) + + bs, l = durs_cums_ends.size() + n_formants = values.size(1) + dcs = durs_cums_starts[:, None, :].expand(bs, n_formants, l) + dce = durs_cums_ends[:, None, :].expand(bs, n_formants, l) + + values_sums = (torch.gather(values_cums, 2, dce) - torch.gather(values_cums, 2, dcs)).float() + values_nelems = (torch.gather(values_nonzero_cums, 2, dce) - torch.gather(values_nonzero_cums, 2, dcs)).float() + + avg = torch.where(values_nelems == 0.0, values_nelems, values_sums / values_nelems) + return avg diff --git a/models/tts/delightful_tts/acoustic_model/length_adaptor.py b/models/tts/delightful_tts/acoustic_model/length_adaptor.py new file mode 100644 index 0000000000000000000000000000000000000000..84af51ec024690ca765e5d75e093d5aaf9daf67c --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/length_adaptor.py @@ -0,0 +1,135 @@ +from typing import List, Tuple + +import torch +from torch.nn import Module + +from models.config import AcousticModelConfigType +from models.helpers import tools + +from .variance_predictor import VariancePredictor + + +class LengthAdaptor(Module): + r"""DEPRECATED: The LengthAdaptor module is used to adjust the duration of phonemes. + It contains a dedicated duration predictor and methods to upsample the input features to match predicted durations. + + Args: + model_config (AcousticModelConfigType): The model configuration object containing model parameters. + """ + + def __init__( + self, + model_config: AcousticModelConfigType, + ): + super().__init__() + # Initialize the duration predictor + self.duration_predictor = VariancePredictor( + channels_in=model_config.encoder.n_hidden, + channels=model_config.variance_adaptor.n_hidden, + channels_out=1, + kernel_size=model_config.variance_adaptor.kernel_size, + p_dropout=model_config.variance_adaptor.p_dropout, + ) + + def length_regulate( + self, + x: torch.Tensor, + duration: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Regulates the length of the input tensor using the duration tensor. + + Args: + x (torch.Tensor): The input tensor. + duration (torch.Tensor): The tensor containing duration for each time step in x. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The regulated output tensor and the tensor containing the length of each sequence in the batch. + """ + output = torch.jit.annotate(List[torch.Tensor], []) + mel_len = torch.jit.annotate(List[int], []) + max_len = 0 + for batch, expand_target in zip(x, duration): + expanded = self.expand(batch, expand_target) + if expanded.shape[0] > max_len: + max_len = expanded.shape[0] + output.append(expanded) + mel_len.append(expanded.shape[0]) + output = tools.pad(output, max_len) + return output, torch.tensor(mel_len, dtype=torch.int64) + + def expand(self, batch: torch.Tensor, predicted: torch.Tensor) -> torch.Tensor: + r"""Expands the input tensor based on the predicted values. + + Args: + batch (torch.Tensor): The input tensor. + predicted (torch.Tensor): The tensor containing predicted expansion factors. + + Returns: + torch.Tensor: The expanded tensor. + """ + out = torch.jit.annotate(List[torch.Tensor], []) + for i, vec in enumerate(batch): + expand_size = predicted[i].item() + out.append(vec.expand(max(int(expand_size), 0), -1)) + return torch.cat(out, 0) + + def upsample_train( + self, + x: torch.Tensor, + x_res: torch.Tensor, + duration_target: torch.Tensor, + embeddings: torch.Tensor, + src_mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Upsamples the input tensor during training using ground truth durations. + + Args: + x (torch.Tensor): The input tensor. + x_res (torch.Tensor): Another input tensor for duration prediction. + duration_target (torch.Tensor): The ground truth durations tensor. + embeddings (torch.Tensor): The tensor containing phoneme embeddings. + src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, log duration prediction, and upsampled embeddings. + """ + x_res = x_res.detach() + log_duration_prediction = self.duration_predictor( + x_res, + src_mask, + ) # type: torch.Tensor + x, _ = self.length_regulate(x, duration_target) + embeddings, _ = self.length_regulate(embeddings, duration_target) + return x, log_duration_prediction, embeddings + + def upsample( + self, + x: torch.Tensor, + x_res: torch.Tensor, + src_mask: torch.Tensor, + embeddings: torch.Tensor, + control: float, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Upsamples the input tensor during inference. + + Args: + x (torch.Tensor): The input tensor. + x_res (torch.Tensor): Another input tensor for duration prediction. + src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res. + embeddings (torch.Tensor): The tensor containing phoneme embeddings. + control (float): A control parameter for pitch regulation. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, approximated duration, and upsampled embeddings. + """ + log_duration_prediction = self.duration_predictor( + x_res, + src_mask, + ) + duration_rounded = torch.clamp( + (torch.round(torch.exp(log_duration_prediction) - 1) * control), + min=0, + ) + x, _ = self.length_regulate(x, duration_rounded) + embeddings, _ = self.length_regulate(embeddings, duration_rounded) + return x, duration_rounded, embeddings diff --git a/models/tts/delightful_tts/acoustic_model/mas.py b/models/tts/delightful_tts/acoustic_model/mas.py new file mode 100644 index 0000000000000000000000000000000000000000..34a4b99f81e43d6f5b66fba788400a67bd7a567a --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/mas.py @@ -0,0 +1,100 @@ +from numba import njit, prange +import numpy as np + +# TODO: +# Check this: https://github.com/archinetai/aligner-pytorch +# and this: https://github.com/resemble-ai/monotonic_align/tree/master + + +# TODO: Don't see any performance improvement with numba +@njit(fastmath=True) +def mas_width1(attn_map: np.ndarray) -> np.ndarray: + r"""Applies a Monotonic Alignments Shrink (MAS) operation with a hard-coded width of 1 to an attention map. + Mas with hardcoded width=1 + Essentially, it produces optimal alignments based on previous attention distribution. + + Args: + attn_map (np.ndarray): The original attention map, a 2D numpy array where rows correspond to mel bins and columns to text bins. + + Returns: + opt (np.ndarray): Returns the optimal attention map after applying the MAS operation. + """ + # assumes mel x text + # Create a placeholder for the output + opt = np.zeros_like(attn_map) + + # Convert the attention map to log scale for stability + attn_map = np.log(attn_map) + + # Initialize the first row of attention map appropriately + attn_map[0, 1:] = -np.inf + + # Initialize log_p with the first row of attention map + log_p = np.zeros_like(attn_map) + log_p[0, :] = attn_map[0, :] + + # Placeholder to remember the previous indices for backtracking later + prev_ind = np.zeros_like(attn_map, dtype=np.int64) + + # Compute the log probabilities based on previous attention distribution + for i in range(1, attn_map.shape[0]): + for j in range(attn_map.shape[1]): # for each text dim + prev_log = log_p[i - 1, j] + prev_j = j + + # Compare with left (j-1) pixel and update if the left pixel has larger log probability + if j - 1 >= 0 and log_p[i - 1, j - 1] >= log_p[i - 1, j]: + prev_log = log_p[i - 1, j - 1] + prev_j = j - 1 + + log_p[i, j] = attn_map[i, j] + prev_log + + # Store the position of maximum cumulative log probability + prev_ind[i, j] = prev_j + + # Backtrack to retrieve the path of attention with maximum cumulative log probability + curr_text_idx = attn_map.shape[1] - 1 + for i in range(attn_map.shape[0] - 1, -1, -1): + opt[i, curr_text_idx] = 1 + curr_text_idx = prev_ind[i, curr_text_idx] + + # Mark the first position of the optimal path + opt[0, curr_text_idx] = 1 + return opt + + +@njit(parallel=True) +def b_mas( + b_attn_map: np.ndarray, + in_lens: np.ndarray, + out_lens: np.ndarray, + width: int=1) -> np.ndarray: + r"""Applies Monotonic Alignments Shrink (MAS) operation in parallel to the batches of an attention map. + It uses the `mas_width1` function internally to perform MAS operation. + + Args: + b_attn_map (np.ndarray): The batched attention map; a 3D array where the first dimension is the batch size, second dimension corresponds to source length, and third dimension corresponds to target length. + in_lens (np.ndarray): Lengths of sequences in the input batch. + out_lens (np.ndarray): Lengths of sequences in the output batch. + width (int, optional): The width for the MAS operation. Defaults to 1. + + Raises: + AssertionError: If width is not equal to 1. This function currently supports only width of 1. + + Returns: + np.ndarray: The batched attention map after applying the MAS operation. It has the same dimensions as `b_attn_map`. + """ + # Assert that the width is 1. This function currently supports only width of 1 + assert width == 1 + attn_out = np.zeros_like(b_attn_map) + + # Loop over each attention map in the batch in parallel + for b in prange(b_attn_map.shape[0]): + # Apply Monotonic Alignments Shrink operation to the b-th attention map in the batch + out = mas_width1(b_attn_map[b, 0, : out_lens[b], : in_lens[b]]) + + # Update the b-th attention map in the output with the result of MAS operation + attn_out[b, 0, : out_lens[b], : in_lens[b]] = out + + # Return the batched attention map after applying the MAS operation + return attn_out diff --git a/models/tts/delightful_tts/acoustic_model/phoneme_prosody_predictor.py b/models/tts/delightful_tts/acoustic_model/phoneme_prosody_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..8b1045726185caf5f2d2d2fef5ea71acaecd4529 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/phoneme_prosody_predictor.py @@ -0,0 +1,102 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import AcousticModelConfigType +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.conv_blocks import ConvTransposed + + +class PhonemeProsodyPredictor(Module): + r"""A class to define the Phoneme Prosody Predictor. + + In linguistics, prosody (/ˈprɒsədi, ˈprɒzədi/) is the study of elements of speech that are not individual phonetic segments (vowels and consonants) but which are properties of syllables and larger units of speech, including linguistic functions such as intonation, stress, and rhythm. Such elements are known as suprasegmentals. + + [Wikipedia Prosody (linguistics)](https://en.wikipedia.org/wiki/Prosody_(linguistics)) + + This prosody predictor is non-parallel and is inspired by the **work of Du et al., 2021 ?**. It consists of + multiple convolution transpose, Leaky ReLU activation, LayerNorm, and dropout layers, followed by a + linear transformation to generate the final output. + + Args: + model_config (AcousticModelConfigType): Configuration object with model parameters. + phoneme_level (bool): A flag to decide whether to use phoneme level bottleneck size. + leaky_relu_slope (float): The negative slope of LeakyReLU activation function. + """ + + def __init__( + self, + model_config: AcousticModelConfigType, + phoneme_level: bool, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + + # Get the configuration + self.d_model = model_config.encoder.n_hidden + kernel_size = model_config.reference_encoder.predictor_kernel_size + dropout = model_config.encoder.p_dropout + + # Decide on the bottleneck size based on phoneme level flag + bottleneck_size = ( + model_config.reference_encoder.bottleneck_size_p + if phoneme_level + else model_config.reference_encoder.bottleneck_size_u + ) + + # Define the layers + self.layers = nn.ModuleList( + [ + ConvTransposed( + self.d_model, + self.d_model, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.LayerNorm( + self.d_model, + ), + nn.Dropout(dropout), + ConvTransposed( + self.d_model, + self.d_model, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.LayerNorm( + self.d_model, + ), + nn.Dropout(dropout), + ], + ) + + # Output bottleneck layer + self.predictor_bottleneck = nn.Linear( + self.d_model, + bottleneck_size, + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the prosody predictor. + + Args: + x (torch.Tensor): A 3-dimensional tensor `[B, src_len, d_model]`. + mask (torch.Tensor): A 2-dimensional tensor `[B, src_len]`. + + Returns: + torch.Tensor: A 3-dimensional tensor `[B, src_len, 2 * d_model]`. + """ + # Expand the mask tensor's dimensions from [B, src_len] to [B, src_len, 1] + mask = mask.unsqueeze(2) + + # Pass the input through the layers + for layer in self.layers: + x = layer(x) + + # Apply mask + x = x.masked_fill(mask, 0.0) + + # Final linear transformation + return self.predictor_bottleneck(x) diff --git a/models/tts/delightful_tts/acoustic_model/pitch_adaptor_conv.py b/models/tts/delightful_tts/acoustic_model/pitch_adaptor_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..32f12110e38c9c30e12d40064a0b2564998a6839 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/pitch_adaptor_conv.py @@ -0,0 +1,179 @@ +from typing import Tuple + +import torch +from torch import nn + +from .helpers import average_over_durations +from .variance_predictor import VariancePredictor + + +class PitchAdaptorConv(nn.Module): + """The PitchAdaptorConv class is a pitch adaptor network in the model. + Updated version of the PitchAdaptorConv uses the conv embeddings for the pitch. + + Args: + channels_in (int): Number of in channels for conv layers. + channels_out (int): Number of out channels. + kernel_size (int): Size the kernel for the conv layers. + dropout (float): Probability of dropout. + leaky_relu_slope (float): Slope for the leaky relu. + emb_kernel_size (int): Size the kernel for the pitch embedding. + + Inputs: inputs, mask + - **inputs** (batch, time1, dim): Tensor containing input vector + - **target** (batch, 1, time2): Tensor containing the pitch target + - **dr** (batch, time1): Tensor containing aligner durations vector + - **mask** (batch, time1): Tensor containing indices to be masked + Returns: + - **pitch prediction** (batch, 1, time1): Tensor produced by pitch predictor + - **pitch embedding** (batch, channels, time1): Tensor produced pitch adaptor + - **average pitch target(train only)** (batch, 1, time1): Tensor produced after averaging over durations + + """ + + def __init__( + self, + channels_in: int, + channels_hidden: int, + channels_out: int, + kernel_size: int, + dropout: float, + leaky_relu_slope: float, + emb_kernel_size: int, + ): + super().__init__() + self.pitch_predictor = VariancePredictor( + channels_in=channels_in, + channels=channels_hidden, + channels_out=channels_out, + kernel_size=kernel_size, + p_dropout=dropout, + leaky_relu_slope=leaky_relu_slope, + ) + self.pitch_emb = nn.Conv1d( + 1, + channels_hidden, + kernel_size=emb_kernel_size, + padding=int((emb_kernel_size - 1) / 2), + ) + + def get_pitch_embedding_train( + self, + x: torch.Tensor, + target: torch.Tensor, + dr: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Function is used during training to get the pitch prediction, average pitch target, + and pitch embedding. + + Args: + x (torch.Tensor): A 3D tensor of shape [B, T_src, C] where B is the batch size, + T_src is the source sequence length, and C is the number of channels. + target (torch.Tensor): A 3D tensor of shape [B, 1, T_max2] where B is the batch size, + T_max2 is the maximum target sequence length. + dr (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the durations. + mask (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the mask. + + Returns: + pitch_pred (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the pitch prediction. + avg_pitch_target (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the average pitch target. + pitch_emb (torch.Tensor): A 3D tensor of shape [B, C, T_src] where B is the batch size, + C is the number of channels, T_src is the source sequence length. The values represent the pitch embedding. + Shapes: + x: :math: `[B, T_src, C]` + target: :math: `[B, 1, T_max2]` + dr: :math: `[B, T_src]` + mask: :math: `[B, T_src]` + """ + pitch_pred = self.pitch_predictor.forward(x, mask) + pitch_pred = pitch_pred.unsqueeze(1) + + avg_pitch_target = average_over_durations(target, dr) + pitch_emb = self.pitch_emb(avg_pitch_target) + + return pitch_pred, avg_pitch_target, pitch_emb + + def add_pitch_embedding_train( + self, + x: torch.Tensor, + target: torch.Tensor, + dr: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Add pitch embedding during training. + + This method calculates the pitch embedding and adds it to the input tensor 'x'. + It also returns the predicted pitch and the average target pitch. + + Args: + x (torch.Tensor): The input tensor to which the pitch embedding will be added. + target (torch.Tensor): The target tensor used in the pitch embedding calculation. + dr (torch.Tensor): The duration tensor used in the pitch embedding calculation. + mask (torch.Tensor): The mask tensor used in the pitch embedding calculation. + + Returns: + x (torch.Tensor): The input tensor with added pitch embedding. + pitch_pred (torch.Tensor): The predicted pitch tensor. + avg_pitch_target (torch.Tensor): The average target pitch tensor. + """ + pitch_pred, avg_pitch_target, pitch_emb = self.get_pitch_embedding_train( + x=x, + target=target.unsqueeze(1), + dr=dr, + mask=mask, + ) + x_pitch = x + pitch_emb.transpose(1, 2) + return x_pitch, pitch_pred, avg_pitch_target + + def get_pitch_embedding( + self, + x: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Function is used during inference to get the pitch embedding and pitch prediction. + + Args: + x (torch.Tensor): A 3D tensor of shape [B, T_src, C] where B is the batch size, + T_src is the source sequence length, and C is the number of channels. + mask (torch.Tensor): A 2D tensor of shape [B, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the mask. + + Returns: + pitch_emb_pred (torch.Tensor): A 3D tensor of shape [B, C, T_src] where B is the batch size, + C is the number of channels, T_src is the source sequence length. The values represent the pitch embedding. + pitch_pred (torch.Tensor): A 3D tensor of shape [B, 1, T_src] where B is the batch size, + T_src is the source sequence length. The values represent the pitch prediction. + """ + pitch_pred = self.pitch_predictor.forward(x, mask) + pitch_pred = pitch_pred.unsqueeze(1) + + pitch_emb_pred = self.pitch_emb(pitch_pred) + return pitch_emb_pred, pitch_pred + + def add_pitch_embedding( + self, + x: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Add pitch embedding during inference. + + This method calculates the pitch embedding and adds it to the input tensor 'x'. + It also returns the predicted pitch. + + Args: + x (torch.Tensor): The input tensor to which the pitch embedding will be added. + mask (torch.Tensor): The mask tensor used in the pitch embedding calculation. + pitch_transform (Callable): A function to transform the pitch prediction. + + Returns: + x (torch.Tensor): The input tensor with added pitch embedding. + pitch_pred (torch.Tensor): The predicted pitch tensor. + """ + pitch_emb_pred, pitch_pred = self.get_pitch_embedding(x, mask) + x_pitch = x + pitch_emb_pred.transpose(1, 2) + return x_pitch, pitch_pred diff --git a/models/tts/delightful_tts/acoustic_model/tests/__init__.py b/models/tts/delightful_tts/acoustic_model/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/tts/delightful_tts/acoustic_model/tests/mocks/speakers.json b/models/tts/delightful_tts/acoustic_model/tests/mocks/speakers.json new file mode 100644 index 0000000000000000000000000000000000000000..ec9c523d0dce128720d7f9d2436934a5a52a3247 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/mocks/speakers.json @@ -0,0 +1,12 @@ +{ + "Speaker_1": 0, + "Speaker_2": 1, + "Speaker_3": 2, + "Speaker_4": 3, + "Speaker_5": 4, + "Speaker_6": 5, + "Speaker_7": 6, + "Speaker_8": 7, + "Speaker_9": 8, + "Speaker_10": 9 +} \ No newline at end of file diff --git a/models/tts/delightful_tts/acoustic_model/tests/mocks/stats.json b/models/tts/delightful_tts/acoustic_model/tests/mocks/stats.json new file mode 100644 index 0000000000000000000000000000000000000000..6a12abb9aad074831b27a56fe285ce9885d84b0d --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/mocks/stats.json @@ -0,0 +1,3 @@ +{ + "pitch": [85.0, 255.0] +} \ No newline at end of file diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_accoustic_model.py b/models/tts/delightful_tts/acoustic_model/tests/test_accoustic_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5dca8bddaf402ccb289a865f88304e40307e1cbf --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_accoustic_model.py @@ -0,0 +1,267 @@ +import unittest + +import torch +from torch.utils.data import DataLoader + +from models.config import AcousticENModelConfig +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig + +# TODO: profile deeply the memory usage +# from torch.profiler import profile, record_function, ProfilerActivity +from models.helpers import tools +from models.helpers.initializer import ( + get_test_configs, + init_acoustic_model, + init_forward_trains_params, +) +from models.helpers.tools import get_mask_from_lengths +from models.tts.delightful_tts.acoustic_model.acoustic_model import AcousticModel +from training.datasets.libritts_dataset_acoustic import LibriTTSDatasetAcoustic +from training.loss.fast_speech_2_loss_gen import FastSpeech2LossGen + + +# AcousticModel test +# Integration test +class TestAcousticModel(unittest.TestCase): + def setUp(self): + # TODO: optimize the model, so that it can be tested with srink_factor=1 + # Get config with srink_factor=4 + # Memory error with srink_factor =< 2 + # Probably because of the size of the model, probably memory leak + # Tried to profile the memory usage, but it didn't help + ( + self.preprocess_config, + self.model_config, + self.acoustic_pretraining_config, + ) = get_test_configs(srink_factor=4) + + # Based on speaker.json mock + n_speakers = 10 + + # Add AcousticModel instance + self.acoustic_model, _ = init_acoustic_model( + self.preprocess_config, + self.model_config, + n_speakers, + ) + + # Generate mock data for the forward pass + self.forward_train_params = init_forward_trains_params( + self.model_config, + self.acoustic_pretraining_config, + self.preprocess_config, + n_speakers, + ) + + def test_get_embeddings(self): + # Generate masks for padding positions in the source sequences and mel sequences + # src_mask: Tensor containing the masks for padding positions in the source sequences. Shape: [1, batch_size] + src_mask = tools.get_mask_from_lengths(self.forward_train_params.src_lens) + + # token_embeddings: Tensor containing the input sequences. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim] + # embeddings: Tensor containing the embeddings. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim + lang_embed_dim] + token_embeddings, embeddings = self.acoustic_model.get_embeddings( + token_idx=self.forward_train_params.x, + speaker_idx=self.forward_train_params.speakers, + src_mask=src_mask, + lang_idx=self.forward_train_params.langs, + ) + + self.assertEqual( + token_embeddings.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.encoder.n_hidden, + ], + ), + ) + self.assertEqual( + embeddings.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.speaker_embed_dim + + self.model_config.lang_embed_dim, + ], + ), + ) + + def test_forward_train(self): + preprocess_config = PreprocessingConfig("english_only") + model_config = AcousticENModelConfig() + + acoustic_model = AcousticModel( + preprocess_config, + model_config, + n_speakers=5392, + ) + + dataset = LibriTTSDatasetAcoustic( + root="datasets_cache/LIBRITTS", + lang="en", + cache=False, + cache_dir="datasets_cache", + mem_cache=False, + url="train-clean-100", + ) + + train_loader = DataLoader( + dataset, + batch_size=1, + num_workers=2, + persistent_workers=True, + pin_memory=True, + shuffle=False, + collate_fn=dataset.collate_fn, + ) + + loss = FastSpeech2LossGen( + bin_warmup=False, + ) + + for batch in train_loader: + ( + _, + _, + speakers, + texts, + src_lens, + mels, + pitches, + _, + mel_lens, + langs, + attn_priors, + _, + energies, + ) = batch + result = acoustic_model.forward_train( + x=texts, + speakers=speakers, + src_lens=src_lens, + mels=mels, + mel_lens=mel_lens, + pitches=pitches, + langs=langs, + attn_priors=attn_priors, + energies=energies, + ) + break + + src_mask = get_mask_from_lengths(src_lens) + mel_mask = get_mask_from_lengths(mel_lens) + + y_pred = result["y_pred"] + log_duration_prediction = result["log_duration_prediction"] + p_prosody_ref = result["p_prosody_ref"] + p_prosody_pred = result["p_prosody_pred"] + pitch_prediction = result["pitch_prediction"] + energy_pred = result["energy_pred"] + energy_target = result["energy_target"] + + loss_out = loss.forward( + src_masks=src_mask, + mel_masks=mel_mask, + mel_targets=mels, + mel_predictions=y_pred, + log_duration_predictions=log_duration_prediction, + u_prosody_ref=result["u_prosody_ref"], + u_prosody_pred=result["u_prosody_pred"], + p_prosody_ref=p_prosody_ref, + p_prosody_pred=p_prosody_pred, + pitch_predictions=pitch_prediction, + p_targets=result["pitch_target"], + durations=result["attn_hard_dur"], + attn_logprob=result["attn_logprob"], + attn_soft=result["attn_soft"], + attn_hard=result["attn_hard"], + src_lens=src_lens, + mel_lens=mel_lens, + energy_pred=energy_pred, + energy_target=energy_target, + step=1000, + ) + + self.assertIsInstance(result, dict) + self.assertIsInstance(loss_out, tuple) + self.assertEqual(len(result), 14) + + def test_average_utterance_prosody(self): + u_prosody_pred = torch.randn(2, 5, self.model_config.encoder.n_hidden) + src_mask = torch.tensor( + [[False, True, True, True, True], [False, False, True, True, True]], + ) + + averaged_prosody_pred = self.acoustic_model.average_utterance_prosody( + u_prosody_pred=u_prosody_pred, + src_mask=src_mask, + ) + + self.assertEqual( + averaged_prosody_pred.shape, + torch.Size([2, 1, self.model_config.encoder.n_hidden]), + ) + + def test_forward(self): + self.preprocess_config = PreprocessingConfig( + language="english_only", + sampling_rate=44100, + ) + self.model_config = AcousticENModelConfig() + + acoustic_model = AcousticModel( + self.preprocess_config, + self.model_config, + n_speakers=5392, + ) + + dataset = LibriTTSDatasetAcoustic( + root="datasets_cache/LIBRITTS", + lang="en", + cache=False, + cache_dir="datasets_cache", + mem_cache=False, + url="train-clean-100", + ) + + train_loader = DataLoader( + dataset, + batch_size=2, + num_workers=2, + persistent_workers=True, + pin_memory=True, + shuffle=False, + collate_fn=dataset.collate_fn, + ) + for batch in train_loader: + ( + _, + _, + speakers, + texts, + _, + _, + _, + _, + _, + langs, + _, + _, + _, + ) = batch + x = acoustic_model.forward( + x=texts, + speakers=speakers, + langs=langs, + d_control=0.5, + ) + break + + self.assertIsInstance(x, torch.Tensor) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_aligner.py b/models/tts/delightful_tts/acoustic_model/tests/test_aligner.py new file mode 100644 index 0000000000000000000000000000000000000000..f1794f7db6c9098fd22f1331dec28de9a6a953f7 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_aligner.py @@ -0,0 +1,103 @@ +import unittest + +import torch + +from models.config import ( + AcousticENModelConfig, + AcousticPretrainingConfig, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.tts.delightful_tts.acoustic_model.aligner import Aligner + + +# It checks for most of the acoustic model code +# Here you can understand the input and output shapes of the Aligner +# Integration test +class TestAligner(unittest.TestCase): + def setUp(self): + self.acoustic_pretraining_config = AcousticPretrainingConfig() + self.model_config = AcousticENModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + self.aligner = Aligner( + d_enc_in=self.model_config.encoder.n_hidden, + d_dec_in=self.preprocess_config.stft.n_mel_channels, + d_hidden=self.model_config.encoder.n_hidden, + ) + + def test_forward(self): + x_res = torch.randn(1, 11, self.model_config.encoder.n_hidden) + mels = torch.randn(1, self.preprocess_config.stft.n_mel_channels, 58) + src_lens = torch.tensor([11]) + mel_lens = torch.tensor([58]) + src_mask = torch.zeros(11).bool() + attn_prior = torch.randn(1, 11, 58) + + attn_logprob, attn_soft, attn_hard, attn_hard_dur = self.aligner( + enc_in=x_res.permute((0, 2, 1)), + dec_in=mels, + enc_len=src_lens, + dec_len=mel_lens, + enc_mask=src_mask, + attn_prior=attn_prior, + ) + + self.assertIsInstance(attn_logprob, torch.Tensor) + self.assertIsInstance(attn_soft, torch.Tensor) + self.assertIsInstance(attn_hard, torch.Tensor) + self.assertIsInstance(attn_hard_dur, torch.Tensor) + + def test_binarize_attention_parallel(self): + aligner = Aligner( + d_enc_in=10, + d_dec_in=10, + d_hidden=20, + ) + batch_size = 5 + max_mel_len = 10 + max_text_len = 15 + + attn = torch.randn( + batch_size, + 1, + max_mel_len, + max_text_len, + ) + in_lens = torch.randint( + 1, + max_mel_len, + (batch_size,), + ) + out_lens = torch.randint( + 1, + max_text_len, + (batch_size,), + ) + + binarized_attention = aligner.binarize_attention_parallel( + attn, + in_lens, + out_lens, + ) + + self.assertIsInstance(binarized_attention, torch.Tensor) + + # Assert the shape of binarized_attention + self.assertEqual( + binarized_attention.shape, + torch.Size( + [ + batch_size, + 1, + max_mel_len, + max_text_len, + ], + ), + ) + self.assertEqual(binarized_attention.shape, attn.shape) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_embedding.py b/models/tts/delightful_tts/acoustic_model/tests/test_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..387b30108238e29fcce8136bf40c1a7d92660b44 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_embedding.py @@ -0,0 +1,52 @@ +import unittest + +import torch + +from models.tts.delightful_tts.acoustic_model.embedding import Embedding + + +class TestEmbedding(unittest.TestCase): + def setUp(self): + self.embedding = Embedding( + num_embeddings=100, + embedding_dim=50, + ) + + def test_forward_output_shape(self): + # Generate a tensor of indices to lookup in the embedding + idx = torch.randint( + low=0, + high=100, + size=(10, 20), + ) # for a sequence of length 20 and batch size 10 + + # Test the forward function + output = self.embedding(idx) + + # Check the output's shape is as expected + self.assertEqual(output.shape, (10, 20, 50)) + + def test_forward_output_values(self): + idx = torch.LongTensor([[0, 50], [99, 1]]) # Indices to lookup in the embedding + + output = self.embedding(idx) + + # Check the values returned by forward function match the expected embeddings + self.assertTrue(torch.all(output[0, 0] == self.embedding.embeddings[0])) + self.assertTrue(torch.all(output[1, 1] == self.embedding.embeddings[1])) + + def test_dtype(self): + idx = torch.randint( + low=0, + high=100, + size=(10, 20), + ) # some example indices + + output = self.embedding(idx) + + # Check the data type of output + self.assertEqual(output.dtype, torch.float32) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_energy_adaptor.py b/models/tts/delightful_tts/acoustic_model/tests/test_energy_adaptor.py new file mode 100644 index 0000000000000000000000000000000000000000..efc513f19a2b682bed9e5ffaf378a2cbac7a2a5a --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_energy_adaptor.py @@ -0,0 +1,116 @@ +import unittest + +import torch + +from models.tts.delightful_tts.acoustic_model.energy_adaptor import EnergyAdaptor + + +class TestEnergyAdaptor(unittest.TestCase): + def setUp(self): + # Initialize common parameters for testing + self.batch_size = 1 + self.seq_length = 11 + self.target_length = 58 + self.channels_in = 58 + self.channels_hidden = 58 + self.channels_out = 1 + self.kernel_size = 5 + self.dropout = 0.1 + self.leaky_relu_slope = 0.2 + self.emb_kernel_size = 3 + + # Initialize the EnergyAdaptor module + self.energy_adaptor = EnergyAdaptor( + channels_in=self.channels_in, + channels_hidden=self.channels_hidden, + channels_out=self.channels_out, + kernel_size=self.kernel_size, + dropout=self.dropout, + leaky_relu_slope=self.leaky_relu_slope, + emb_kernel_size=self.emb_kernel_size, + ) + + # Create sample input tensors + self.inputs = torch.randn(self.batch_size, self.seq_length, self.channels_in) + self.target = torch.randn(self.batch_size, 1, self.target_length) + self.dr = torch.tensor([[ 5., 5., 5., 5., 4., 5., 5., 4., 5., 5., 10.]]) + self.mask = torch.randint(1, self.seq_length, (self.batch_size, self.seq_length)).bool() + + def test_get_energy_embedding_train(self): + # Test get_energy_embedding_train method + energy_pred, avg_energy_target, energy_emb = self.energy_adaptor.get_energy_embedding_train( + x=self.inputs, + target=self.target, + dr=self.dr, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(energy_pred.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(avg_energy_target.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(energy_emb.shape, (self.batch_size, self.channels_hidden, self.seq_length)) + + def test_add_energy_embedding_train(self): + # Test add_energy_embedding_train method + ( + x_with_energy, + energy_pred, + avg_energy_target, + ) = self.energy_adaptor.add_energy_embedding_train( + x=self.inputs, + target=self.target, + dr=self.dr, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(x_with_energy.shape, self.inputs.shape) + self.assertEqual(energy_pred.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(avg_energy_target.shape, (self.batch_size, 1, self.seq_length)) + + def test_get_energy_embedding(self): + # Initialize the EnergyAdaptor module + energy_adaptor = EnergyAdaptor( + channels_in=self.channels_in, + channels_hidden=self.channels_hidden, + channels_out=self.channels_out, + kernel_size=self.kernel_size, + dropout=self.dropout, + leaky_relu_slope=self.leaky_relu_slope, + emb_kernel_size=self.emb_kernel_size, + ) + + # Test get_energy_embedding method + energy_emb_pred, energy_pred = energy_adaptor.get_energy_embedding( + x=self.inputs, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(energy_emb_pred.shape, (self.batch_size, self.channels_hidden, self.seq_length)) + self.assertEqual(energy_pred.shape, (self.batch_size, 1, self.seq_length)) + + def test_add_energy_embedding(self): + # Initialize the EnergyAdaptor module + energy_adaptor = EnergyAdaptor( + channels_in=self.channels_in, + channels_hidden=self.channels_hidden, + channels_out=self.channels_out, + kernel_size=self.kernel_size, + dropout=self.dropout, + leaky_relu_slope=self.leaky_relu_slope, + emb_kernel_size=self.emb_kernel_size, + ) + + # Test add_energy_embedding method + x_with_energy, energy_pred = energy_adaptor.add_energy_embedding( + x=self.inputs, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(x_with_energy.shape, self.inputs.shape) + self.assertEqual(energy_pred.shape, (self.batch_size, 1, self.seq_length)) + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_length_adaptor.py b/models/tts/delightful_tts/acoustic_model/tests/test_length_adaptor.py new file mode 100644 index 0000000000000000000000000000000000000000..88735f5667772684b9dad3ce18bf35169cee6fa3 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_length_adaptor.py @@ -0,0 +1,140 @@ +import unittest +from unittest.mock import Mock + +import torch + +from models.tts.delightful_tts.acoustic_model.length_adaptor import LengthAdaptor + + +class TestLengthAdaptor(unittest.TestCase): + def setUp(self): + mock_model_config = Mock() + + # Attributes for model_config.encoder + mock_model_config.encoder.n_hidden = 512 + + # Attributes for model_config.variance_adaptor + mock_model_config.variance_adaptor.n_hidden = 512 + mock_model_config.variance_adaptor.kernel_size = 5 + mock_model_config.variance_adaptor.p_dropout = 0.5 + + self.model_config = mock_model_config + + self.length_adaptor = LengthAdaptor( + self.model_config, + ) + self.batch_size = 2 + self.seq_length = 5 + self.n_hidden = 512 # should match cls.model_config.encoder.n_hidden + + self.x = torch.rand( + self.batch_size, + self.seq_length, + self.n_hidden, + ) + self.x_res = torch.rand( + self.batch_size, + self.seq_length, + self.n_hidden, + ) + self.src_mask = torch.ones( + self.batch_size, + self.seq_length, + dtype=torch.bool, + ) + + def test_length_regulate(self): + duration = torch.full( + (self.batch_size, self.seq_length), + fill_value=2.0, + ) + output, mel_len = self.length_adaptor.length_regulate(self.x, duration) + + self.assertTrue(torch.is_tensor(output)) + self.assertTrue(torch.is_tensor(mel_len)) + + def test_expand(self): + predicted = torch.randint( + low=0, + high=2, + size=(self.seq_length,), + ) + out = self.length_adaptor.expand(self.x[0], predicted) + + self.assertTrue(torch.is_tensor(out)) + + # Getting the sum of predicted tensor values, which will be the expected dimension size after expand + # Assuming dimension 0 (time-steps) is being expanded + expected_dim_0 = predicted.sum().item() + actual_dim_0 = out.size(0) + + # Check if the size of expanded dimension matches with the sum of predicted values. + self.assertEqual(expected_dim_0, actual_dim_0) + + # Check the size of unexpanded dimensions + self.assertEqual(self.x.size(1), self.seq_length) + self.assertEqual(out.size(1), self.n_hidden) + + def test_upsample_train(self): + duration_target = torch.full( + (self.batch_size, self.seq_length), + fill_value=2.0, + ) + embeddings = torch.rand( + self.batch_size, + self.seq_length, + self.n_hidden, + ) + + x, log_duration_prediction, new_embeddings = self.length_adaptor.upsample_train( + self.x, self.x_res, duration_target, embeddings, self.src_mask, + ) + + self.assertTrue(torch.is_tensor(x)) + self.assertTrue(torch.is_tensor(log_duration_prediction)) + self.assertTrue(torch.is_tensor(new_embeddings)) + + # Check the size of tensor x, log_duration_prediction and new_embeddings + expected_dim_0 = duration_target.sum().item() + + new_size_coef = self.batch_size * self.seq_length + + self.assertEqual(expected_dim_0, x.size(0) * new_size_coef) + self.assertEqual(self.x.size(1), self.seq_length) + self.assertEqual(x.size(2), self.n_hidden) + + self.assertEqual(log_duration_prediction.size(0), self.batch_size) + self.assertEqual(log_duration_prediction.size(1), self.seq_length) + + self.assertEqual(expected_dim_0, new_embeddings.size(0) * new_size_coef) + + self.assertEqual(new_embeddings.size(1), self.seq_length * self.batch_size) + self.assertEqual(new_embeddings.size(2), self.n_hidden) + + def test_upsample(self): + embeddings = torch.rand(self.batch_size, self.seq_length, self.n_hidden) + control = 0.5 + x, duration_rounded, new_embeddings = self.length_adaptor.upsample( + self.x, self.x_res, self.src_mask, embeddings, control, + ) + + self.assertTrue(torch.is_tensor(x)) + self.assertTrue(torch.is_tensor(duration_rounded)) + self.assertTrue(torch.is_tensor(new_embeddings)) + + # Check the size of tensor x, duration_rounded and new_embeddings + self.assertEqual(x.size(0), self.batch_size) + self.assertEqual(self.x.size(1), self.seq_length) + self.assertEqual(x.size(2), self.n_hidden) + + self.assertEqual(duration_rounded.size(0), self.batch_size) + self.assertEqual(duration_rounded.size(1), self.seq_length) + + self.assertEqual(new_embeddings.size(0), self.batch_size) + # Always different? + # self.assertEqual(new_embeddings.size(1), 1) + self.assertEqual(new_embeddings.size(2), self.n_hidden) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_mas.py b/models/tts/delightful_tts/acoustic_model/tests/test_mas.py new file mode 100644 index 0000000000000000000000000000000000000000..fcbcefdeb7c5fb07d724bb8c79a6c1a207732d4f --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_mas.py @@ -0,0 +1,66 @@ +import unittest + +from numba import prange +import numpy as np + +from models.tts.delightful_tts.acoustic_model.mas import b_mas, mas_width1 + + +class TestMasWidth1(unittest.TestCase): + def setUp(self): + # Example attn_map with random number between 0 and 1 + self.attn_map = np.random.rand(5, 5) + + def test_mas_width1(self): + # Test output of mas_width1 function + opt = mas_width1(self.attn_map) + + # Assert opt returned is a numpy ndarray + self.assertIsInstance(opt, np.ndarray) + + # Assert the shapes of input attn_map and output opt are same + self.assertEqual(opt.shape, self.attn_map.shape) + + # Assert opt only contains 0s and 1s (as per function description) + self.assertTrue(np.array_equal(opt, opt.astype(bool))) + + # Assert that at least one entry in opt is 1.0 (since at least one optimal position should exist) + self.assertIn(1.0, opt) + + +class TestBMas(unittest.TestCase): + def setUp(self): + # Create a sample batched attention map for testing + # This sets up a batch of 2 attention maps, each of size 5 by 5 + self.b_attn_map = np.random.rand(2, 1, 5, 5) + # Lengths of sequences in the input batch + self.in_lens = np.array([3, 4]) + # Lengths of sequences in the output batch + self.out_lens = np.array([4, 5]) + + def test_b_mas(self): + # Run the b_mas function taking width = 1 + attn_out = b_mas(self.b_attn_map, self.in_lens, self.out_lens, width=1) + + # Check the output type + self.assertIsInstance(attn_out, np.ndarray) + + # Check if output and input have same shape + self.assertEqual(attn_out.shape, self.b_attn_map.shape) + + # Assert attn_out only contains 0s and 1s. + self.assertTrue(np.array_equal(attn_out, attn_out.astype(bool))) + + # Verify that the first dimension size equals batch size (2) + self.assertEqual(attn_out.shape[0], 2) + + # Verify that the third and fourth dimensions size matches the given out_lens and in_len + for b in prange(attn_out.shape[0]): + self.assertEqual( + np.sum(attn_out[b, 0, : self.out_lens[b], : self.in_lens[b]]), + self.out_lens[b], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_phoneme_prosody_predictor.py b/models/tts/delightful_tts/acoustic_model/tests/test_phoneme_prosody_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..caf929e0169b021fa55016086da77a5ab8e261e1 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_phoneme_prosody_predictor.py @@ -0,0 +1,47 @@ +import unittest +from unittest.mock import Mock + +import torch + +from models.tts.delightful_tts.acoustic_model import PhonemeProsodyPredictor + + +class TestPhonemeProsodyPredictor(unittest.TestCase): + def setUp(self): + model_config = Mock( + encoder=Mock( + n_layers=4, + n_heads=6, + n_hidden=384, + p_dropout=0.1, + kernel_size_conv_mod=7, + kernel_size_depthwise=7, + with_ff=False, + ), + reference_encoder=Mock( + predictor_kernel_size=3, + p_dropout=0.1, + bottleneck_size_p=128, + bottleneck_size_u=256, + ), + ) + self.model = PhonemeProsodyPredictor(model_config, phoneme_level=True) + + def test_forward(self): + x = torch.rand( + 16, + 384, + 384, + ) + mask = torch.zeros(16, 384).bool() + + # Calling model's forward method + out = self.model(x, mask) + + # Check if the output is of expected type and shape + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.size(), (16, 384, 128)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_pitch_adaptor_conv.py b/models/tts/delightful_tts/acoustic_model/tests/test_pitch_adaptor_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..68992460eb937d24ee5c471f3b217f285d2e4d7f --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_pitch_adaptor_conv.py @@ -0,0 +1,102 @@ +import unittest + +import torch + +from models.tts.delightful_tts.acoustic_model.pitch_adaptor_conv import PitchAdaptorConv + + +class TestPitchAdaptorConv(unittest.TestCase): + def setUp(self): + # Initialize common parameters for testing + self.batch_size = 1 + self.seq_length = 11 + self.target_length = 58 + self.channels_in = 58 + self.channels_hidden = 58 + self.channels_out = 1 + self.kernel_size = 5 + self.dropout = 0.1 + self.leaky_relu_slope = 0.2 + self.emb_kernel_size = 3 + + # Initialize the PitchAdaptorConv module + self.pitch_adaptor = PitchAdaptorConv( + channels_in=self.channels_in, + channels_hidden=self.channels_hidden, + channels_out=self.channels_out, + kernel_size=self.kernel_size, + dropout=self.dropout, + leaky_relu_slope=self.leaky_relu_slope, + emb_kernel_size=self.emb_kernel_size, + ) + + # Create sample input tensors + self.inputs = torch.randn(self.batch_size, self.seq_length, self.channels_in) + self.target = torch.randn(self.batch_size, 1, self.target_length) + self.dr = torch.tensor([[ 5., 5., 5., 5., 4., 5., 5., 4., 5., 5., 10.]]) + self.mask = torch.randint(1, self.seq_length, (self.batch_size, self.seq_length)).bool() + + def test_get_pitch_embedding_train(self): + # Test get_pitch_embedding_train method + pitch_pred, avg_pitch_target, pitch_emb = self.pitch_adaptor.get_pitch_embedding_train( + x=self.inputs, + target=self.target, + dr=self.dr, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(pitch_pred.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(avg_pitch_target.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(pitch_emb.shape, (self.batch_size, self.channels_hidden, self.seq_length)) + + def test_add_pitch_embedding_train(self): + inputs = torch.randn(self.batch_size, self.seq_length, self.channels_in) + target = torch.randn(self.batch_size, self.target_length) + dr = torch.tensor([[ 5., 5., 5., 5., 4., 5., 5., 4., 5., 5., 10.]]) + mask = torch.randint(1, self.seq_length, (self.batch_size, self.seq_length)).bool() + + # Test add_pitch_embedding_train method + ( + x_with_pitch, + pitch_pred, + avg_pitch_target, + ) = self.pitch_adaptor.add_pitch_embedding_train( + x=inputs, + target=target, + dr=dr, + mask=mask, + ) + + # Check shapes of output tensors + self.assertEqual(x_with_pitch.shape, self.inputs.shape) + self.assertEqual(pitch_pred.shape, (self.batch_size, 1, self.seq_length)) + self.assertEqual(avg_pitch_target.shape, (self.batch_size, 1, self.seq_length)) + + def test_get_pitch_embedding(self): + # Test get_pitch_embedding method + pitch_emb_pred, pitch_pred = self.pitch_adaptor.get_pitch_embedding( + x=self.inputs, + mask=self.mask, + ) + + # Check shapes of output tensors + self.assertEqual(pitch_emb_pred.shape, (self.batch_size, self.channels_hidden, self.seq_length)) + self.assertEqual(pitch_pred.shape, (self.batch_size, 1, self.seq_length)) + + def test_add_pitch_embedding(self): + inputs = torch.randn(self.batch_size, self.seq_length, self.channels_in) + mask = torch.randint(1, self.seq_length, (self.batch_size, self.seq_length)).bool() + + # Test add_pitch_embedding method + x_with_pitch, pitch_pred = self.pitch_adaptor.add_pitch_embedding( + x=inputs, + mask=mask, + ) + + # Check shapes of output tensors + self.assertEqual(x_with_pitch.shape, self.inputs.shape) + self.assertEqual(pitch_pred.shape, (self.batch_size, 1, self.seq_length)) + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/tests/test_variance_predictor.py b/models/tts/delightful_tts/acoustic_model/tests/test_variance_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..fd0e7ad168fcddc03cf600c2aab7e02760cb4a07 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/tests/test_variance_predictor.py @@ -0,0 +1,71 @@ +import unittest + +import torch + +from models.tts.delightful_tts.acoustic_model.variance_predictor import ( + VariancePredictor, +) + + +class TestVariancePredictor(unittest.TestCase): + def setUp(self): + # Initialize a VariancePredictor instance + self.predictor = VariancePredictor( + channels_in=32, + channels=32, + channels_out=1, + kernel_size=3, + p_dropout=0.5, + ) + + # Assume batch size=3, channels_in=32, sequence_length=32 + self.x = torch.rand((3, 32, 32)) + + # Assume batch size=3, sequence_length=32 + self.mask_dim = (3, 32) + self.zero_mask = torch.ones(self.mask_dim).type(torch.bool) + + def test_forward(self): + # Execute forward propagation + output = self.predictor(self.x, self.zero_mask) + + # Validate output shape + self.assertEqual( + output.shape, self.mask_dim, + ) # Expected shape is (N, T) where N=batch size and T=sequence length + + def test_zero_mask(self): + # Execute forward propagation + output = self.predictor(self.x, self.zero_mask) + + # Validate all returned values are zero, given the mask is all False + self.assertTrue(torch.all(output == 0)) + + def test_ones_mask(self): + # Create a mask of ones (indicating no entries are masked) + ones_mask = torch.ones(self.mask_dim).type(torch.bool) + + # Execute forward propagation + output = self.predictor(self.x, ones_mask) + + # Validate all returned values are not zero given all are True + self.assertFalse(torch.all(output != 0)) + + def test_output_dtype(self): + # Execute forward propagation + output = self.predictor(self.x, self.zero_mask) + + # Check the data type of output + self.assertEqual(output.dtype, torch.float32) + + def test_output_range(self): + # Execute forward propagation + output = self.predictor(self.x, self.zero_mask) + + # Validate the output values are between 0 and 1 + self.assertGreaterEqual(output.min(), 0) + self.assertLessEqual(output.max(), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/acoustic_model/variance_predictor.py b/models/tts/delightful_tts/acoustic_model/variance_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..c57f525d516d6e46cd25b7ad12af6e67ba8be786 --- /dev/null +++ b/models/tts/delightful_tts/acoustic_model/variance_predictor.py @@ -0,0 +1,90 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.conv_blocks import ConvTransposed + + +class VariancePredictor(Module): + r"""Duration and Pitch predictor neural network module in PyTorch. + + It consists of multiple layers, including `ConvTransposed` layers (custom convolution transpose layers from + the `model.conv_blocks` module), LeakyReLU activation functions, Layer Normalization and Dropout layers. + + Constructor for `VariancePredictor` class. + + Args: + channels_in (int): Number of input channels. + channels (int): Number of output channels for ConvTransposed layers and input channels for linear layer. + channels_out (int): Number of output channels for linear layer. + kernel_size (int): Size of the kernel for ConvTransposed layers. + p_dropout (float): Probability of dropout. + + Returns: + torch.Tensor: Output tensor. + """ + + def __init__( + self, + channels_in: int, + channels: int, + channels_out: int, + kernel_size: int, + p_dropout: float, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + + self.layers = nn.ModuleList( + [ + # Convolution transpose layer followed by LeakyReLU, LayerNorm and Dropout + ConvTransposed( + channels_in, + channels, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.LayerNorm( + channels, + ), + nn.Dropout(p_dropout), + # Another "block" of ConvTransposed, LeakyReLU, LayerNorm, and Dropout + ConvTransposed( + channels, + channels, + kernel_size=kernel_size, + padding=(kernel_size - 1) // 2, + ), + nn.LeakyReLU(leaky_relu_slope), + nn.LayerNorm( + channels, + ), + nn.Dropout(p_dropout), + ], + ) + + # Output linear layer + self.linear_layer = nn.Linear( + channels, + channels_out, + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + r"""Forward pass for `VariancePredictor`. + + Args: + x (torch.Tensor): Input tensor. + mask (torch.Tensor): Mask tensor, has the same size as x. + + Returns: + torch.Tensor: Output tensor. + """ + # Sequentially pass the input through all defined layers + # (ConvTransposed -> LeakyReLU -> LayerNorm -> Dropout -> ConvTransposed -> LeakyReLU -> LayerNorm -> Dropout) + for layer in self.layers: + x = layer(x) + x = self.linear_layer(x) + x = x.squeeze(-1) + return x.masked_fill(mask, 0.0) diff --git a/models/tts/delightful_tts/attention/__init__.py b/models/tts/delightful_tts/attention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a82aad690988f50b525174069db1776eebac911 --- /dev/null +++ b/models/tts/delightful_tts/attention/__init__.py @@ -0,0 +1,6 @@ +from .conformer import Conformer +from .conformer_block import ConformerBlock +from .conformer_multi_headed_self_attention import ConformerMultiHeadedSelfAttention +from .multi_head_attention import MultiHeadAttention +from .relative_multi_head_attention import RelativeMultiHeadAttention +from .style_embed_attention import StyleEmbedAttention diff --git a/models/tts/delightful_tts/attention/conformer.py b/models/tts/delightful_tts/attention/conformer.py new file mode 100644 index 0000000000000000000000000000000000000000..a5066d557c596420c56b5a41cafeec360b5f34ce --- /dev/null +++ b/models/tts/delightful_tts/attention/conformer.py @@ -0,0 +1,75 @@ +import torch +from torch import nn +from torch.nn import Module + +from .conformer_block import ConformerBlock + + +class Conformer(Module): + r"""`Conformer` class represents the `Conformer` model which is a sequence-to-sequence model + used in some modern automated speech recognition systems. It is composed of several `ConformerBlocks`. + + Args: + dim (int): The number of expected features in the input. + n_layers (int): The number of `ConformerBlocks` in the Conformer model. + n_heads (int): The number of heads in the multiheaded self-attention mechanism in each `ConformerBlock`. + embedding_dim (int): The dimension of the embeddings. + p_dropout (float): The dropout probability to be used in each `ConformerBlock`. + kernel_size_conv_mod (int): The size of the convolving kernel in the convolution module of each `ConformerBlock`. + with_ff (bool): If True, each `ConformerBlock` uses FeedForward layer inside it. + """ + + def __init__( + self, + dim: int, + n_layers: int, + n_heads: int, + embedding_dim: int, + p_dropout: float, + kernel_size_conv_mod: int, + with_ff: bool, + ): + super().__init__() + self.layer_stack = nn.ModuleList( + [ + ConformerBlock( + dim, + n_heads, + kernel_size_conv_mod=kernel_size_conv_mod, + dropout=p_dropout, + embedding_dim=embedding_dim, + with_ff=with_ff, + ) + for _ in range(n_layers) + ], + ) + + def forward( + self, + x: torch.Tensor, + mask: torch.Tensor, + embeddings: torch.Tensor, + encoding: torch.Tensor, + ) -> torch.Tensor: + r"""Forward Pass of the Conformer block. + + Args: + x (Tensor): Input tensor of shape (batch_size, seq_len, num_features). + mask (Tensor): The mask tensor. + embeddings (Tensor): Embeddings tensor. + encoding (Tensor): The positional encoding tensor. + + Returns: + Tensor: The output tensor of shape (batch_size, seq_len, num_features). + """ + attn_mask = mask.view((mask.shape[0], 1, 1, mask.shape[1])) + attn_mask.to(x.device) + for enc_layer in self.layer_stack: + x = enc_layer( + x, + mask=mask, + slf_attn_mask=attn_mask, + embeddings=embeddings, + encoding=encoding, + ) + return x diff --git a/models/tts/delightful_tts/attention/conformer_block.py b/models/tts/delightful_tts/attention/conformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..d7e5897b54a42e37e72685b6a19326bb26502cff --- /dev/null +++ b/models/tts/delightful_tts/attention/conformer_block.py @@ -0,0 +1,103 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.tts.delightful_tts.conv_blocks import Conv1dGLU + +from .conformer_conv_module import ConformerConvModule +from .conformer_multi_headed_self_attention import ConformerMultiHeadedSelfAttention +from .feed_forward import FeedForward + + +class ConformerBlock(Module): + r"""ConformerBlock class represents a block in the Conformer model architecture. + The block includes a pointwise convolution followed by Gated Linear Units (`GLU`) activation layer (`Conv1dGLU`), + a Conformer self attention layer (`ConformerMultiHeadedSelfAttention`), and optional feed-forward layer (`FeedForward`). + + Args: + d_model (int): The number of expected features in the input. + n_head (int): The number of heads for the multiheaded attention mechanism. + kernel_size_conv_mod (int): The size of the convolving kernel for the convolution module. + embedding_dim (int): The dimension of the embeddings. + dropout (float): The dropout probability. + with_ff (bool): If True, uses FeedForward layer inside ConformerBlock. + """ + + def __init__( + self, + d_model: int, + n_head: int, + kernel_size_conv_mod: int, + embedding_dim: int, + dropout: float, + with_ff: bool, + ): + super().__init__() + self.with_ff = with_ff + self.conditioning = Conv1dGLU( + d_model=d_model, + kernel_size=kernel_size_conv_mod, + padding=kernel_size_conv_mod // 2, + embedding_dim=embedding_dim, + ) + if self.with_ff: + self.ff = FeedForward( + d_model=d_model, + dropout=dropout, + kernel_size=3, + ) + self.conformer_conv_1 = ConformerConvModule( + d_model, + kernel_size=kernel_size_conv_mod, + dropout=dropout, + ) + self.ln = nn.LayerNorm( + d_model, + ) + self.slf_attn = ConformerMultiHeadedSelfAttention( + d_model=d_model, + num_heads=n_head, + dropout_p=dropout, + ) + self.conformer_conv_2 = ConformerConvModule( + d_model, + kernel_size=kernel_size_conv_mod, + dropout=dropout, + ) + + def forward( + self, + x: torch.Tensor, + embeddings: torch.Tensor, + mask: torch.Tensor, + slf_attn_mask: torch.Tensor, + encoding: torch.Tensor, + ) -> torch.Tensor: + r"""Forward pass of the Conformer block. + + Args: + x (Tensor): Input tensor of shape (batch_size, seq_len, num_features). + embeddings (Tensor): Embeddings tensor. + mask (Tensor): The mask tensor. + slf_attn_mask (Tensor): The mask for self-attention layer. + encoding (Tensor): The positional encoding tensor. + + Returns: + Tensor: The output tensor of shape (batch_size, seq_len, num_features). + """ + x = self.conditioning.forward(x, embeddings=embeddings) + if self.with_ff: + x = self.ff(x) + x + x = self.conformer_conv_1(x) + x + res = x + x = self.ln(x) + x, _ = self.slf_attn( + query=x, + key=x, + value=x, + mask=slf_attn_mask, + encoding=encoding, + ) + x = x + res + x = x.masked_fill(mask.unsqueeze(-1), 0) + return self.conformer_conv_2(x) + x diff --git a/models/tts/delightful_tts/attention/conformer_conv_module.py b/models/tts/delightful_tts/attention/conformer_conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..5a60deb8d5d3c485987b2be58aeb3e635d792c1c --- /dev/null +++ b/models/tts/delightful_tts/attention/conformer_conv_module.py @@ -0,0 +1,78 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.helpers import tools +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.conv_blocks import ( + DepthWiseConv1d, + GLUActivation, + PointwiseConv1d, +) + + +class ConformerConvModule(Module): + r"""Conformer Convolution Module class represents a module in the Conformer model architecture. + The module includes a layer normalization, pointwise and depthwise convolutional layers, + Gated Linear Units (GLU) activation, and dropout layer. + + Args: + d_model (int): The number of expected features in the input. + expansion_factor (int): The expansion factor for the hidden layer size in the feed-forward network, default is 2. + kernel_size (int): The size of the convolving kernel, default is 7. + dropout (float): The dropout probability, default is 0.1. + leaky_relu_slope (float): Controls the angle of the negative slope of the LeakyReLU activation, default is `LEAKY_RELU_SLOPE`. + """ + + def __init__( + self, + d_model: int, + expansion_factor: int = 2, + kernel_size: int = 7, + dropout: float = 0.1, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + inner_dim = d_model * expansion_factor + self.ln_1 = nn.LayerNorm(d_model) + self.conv_1 = PointwiseConv1d( + d_model, + inner_dim * 2, + ) + self.conv_act = GLUActivation() + self.depthwise = DepthWiseConv1d( + inner_dim, + inner_dim, + kernel_size=kernel_size, + padding=tools.calc_same_padding(kernel_size)[0], + ) + self.ln_2 = nn.GroupNorm( + 1, + inner_dim, + ) + self.activation = nn.LeakyReLU(leaky_relu_slope) + self.conv_2 = PointwiseConv1d( + inner_dim, + d_model, + ) + self.dropout = nn.Dropout(dropout) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the Conformer conv module. + + Args: + x (Tensor): Input tensor of shape (batch_size, seq_len, num_features). + + Returns: + Tensor: The output tensor of shape (batch_size, seq_len, num_features). + """ + x = self.ln_1(x) + x = x.permute(0, 2, 1) + x = self.conv_1(x) + x = self.conv_act(x) + x = self.depthwise(x) + x = self.ln_2(x) + x = self.activation(x) + x = self.conv_2(x) + x = x.permute(0, 2, 1) + return self.dropout(x) diff --git a/models/tts/delightful_tts/attention/conformer_multi_headed_self_attention.py b/models/tts/delightful_tts/attention/conformer_multi_headed_self_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..b345767764f41b5fbc552472573c97f642815686 --- /dev/null +++ b/models/tts/delightful_tts/attention/conformer_multi_headed_self_attention.py @@ -0,0 +1,65 @@ +from typing import Tuple + +import torch +from torch import nn +from torch.nn import Module + +from .relative_multi_head_attention import RelativeMultiHeadAttention + + +class ConformerMultiHeadedSelfAttention(Module): + """Conformer employ multi-headed self-attention (MHSA) while integrating an important technique from Transformer-XL, + the relative sinusoidal positional encoding scheme. The relative positional encoding allows the self-attention + module to generalize better on different input length and the resulting encoder is more robust to the variance of + the utterance length. Conformer use `prenorm` residual units with dropout which helps training + and regularizing deeper models. + + Args: + d_model (int): The dimension of model + num_heads (int): The number of attention heads. + dropout_p (float): probability of dropout + + Inputs: inputs, mask + - **inputs** (batch, time, dim): Tensor containing input vector + - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked + + Returns: + (batch, time, dim): Tensor produces by relative multi headed self attention module. + """ + + def __init__( + self, + d_model: int, + num_heads: int, + dropout_p: float, + ): + super().__init__() + + # Initialize the RelativeMultiHeadAttention module passing the model dimension and number of attention heads + self.attention = RelativeMultiHeadAttention( + d_model=d_model, num_heads=num_heads, + ) + self.dropout = nn.Dropout(p=dropout_p) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + mask: torch.Tensor, + encoding: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + batch_size, _, _ = key.size() + + # Trim or extend the "encoding" to match the size of key, and repeat this for each input in the batch + encoding = encoding[:, : key.shape[1]] + encoding = encoding.repeat(batch_size, 1, 1) + + # Pass inputs through the RelativeMultiHeadAttention layer, dropout the resulting outputs + outputs, attn = self.attention( + query, key, value, pos_embedding=encoding, mask=mask, + ) + + # Apply dropout to the attention outputs + outputs = self.dropout(outputs) + return outputs, attn diff --git a/models/tts/delightful_tts/attention/feed_forward.py b/models/tts/delightful_tts/attention/feed_forward.py new file mode 100644 index 0000000000000000000000000000000000000000..99ac0c4b6d3ad07c6b1483529a7b11262dcea055 --- /dev/null +++ b/models/tts/delightful_tts/attention/feed_forward.py @@ -0,0 +1,66 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE + + +class FeedForward(Module): + r"""Creates a feed-forward neural network. + The network includes a layer normalization, an activation function (LeakyReLU), and dropout layers. + + Args: + d_model (int): The number of expected features in the input. + kernel_size (int): The size of the convolving kernel for the first conv1d layer. + dropout (float): The dropout probability. + expansion_factor (int, optional): The expansion factor for the hidden layer size in the feed-forward network, default is 4. + leaky_relu_slope (float, optional): Controls the angle of the negative slope of LeakyReLU activation, default is `LEAKY_RELU_SLOPE`. + """ + + def __init__( + self, + d_model: int, + kernel_size: int, + dropout: float, + expansion_factor: int = 4, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ): + super().__init__() + self.dropout = nn.Dropout(dropout) + self.ln = nn.LayerNorm(d_model) + self.conv_1 = nn.Conv1d( + d_model, + d_model * expansion_factor, + kernel_size=kernel_size, + padding=kernel_size // 2, + ) + self.act = nn.LeakyReLU(leaky_relu_slope) + self.conv_2 = nn.Conv1d(d_model * expansion_factor, d_model, kernel_size=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the feed-forward neural network. + + Args: + x (Tensor): Input tensor of shape (batch_size, seq_len, num_features). + + Returns: + Tensor: Output tensor of shape (batch_size, seq_len, num_features). + """ + # Apply layer normalization + x = self.ln(x) + + # Forward pass through the first convolution layer, activation layer and dropout layer + x = x.permute((0, 2, 1)) + x = self.conv_1(x) + x = x.permute((0, 2, 1)) + x = self.act(x) + x = self.dropout(x) + + # Forward pass through the second convolution layer and dropout layer + x = x.permute((0, 2, 1)) + x = self.conv_2(x) + x = x.permute((0, 2, 1)) + x = self.dropout(x) + + # Scale the output by 0.5 (this helps with training stability) + return 0.5 * x diff --git a/models/tts/delightful_tts/attention/multi_head_attention.py b/models/tts/delightful_tts/attention/multi_head_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..a621716983cb97b3e98169bd75246cf28a1fcd4a --- /dev/null +++ b/models/tts/delightful_tts/attention/multi_head_attention.py @@ -0,0 +1,90 @@ +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F + + +class MultiHeadAttention(Module): + r"""A class that implements a Multi-head Attention mechanism. + Multi-head attention allows the model to focus on different positions, + capturing various aspects of the input. + + Args: + query_dim (int): The dimensionality of the query. + key_dim (int): The dimensionality of the key. + num_units (int): The total number of dimensions of the output. + num_heads (int): The number of parallel attention layers (multi-heads). + + Inputs: query, and key + - **query**: Tensor of shape [N, T_q, query_dim] + - **key**: Tensor of shape [N, T_k, key_dim] + + Outputs: + - An output tensor of shape [N, T_q, num_units] + """ + + def __init__( + self, + query_dim: int, + key_dim: int, + num_units: int, + num_heads: int, + ): + super().__init__() + self.num_units = num_units + self.num_heads = num_heads + self.key_dim = key_dim + + self.W_query = nn.Linear( + in_features=query_dim, + out_features=num_units, + bias=False, + ) + self.W_key = nn.Linear(in_features=key_dim, out_features=num_units, bias=False) + self.W_value = nn.Linear( + in_features=key_dim, out_features=num_units, bias=False, + ) + + def forward(self, query: torch.Tensor, key: torch.Tensor) -> torch.Tensor: + r"""Performs the forward pass over input tensors. + + Args: + query (torch.Tensor): The input tensor containing query vectors. + It is expected to have the dimensions [N, T_q, query_dim] + where N is the batch size, T_q is the sequence length of queries, + and query_dim is the dimensionality of a single query vector. + + key (torch.Tensor): The input tensor containing key vectors. + It is expected to have the dimensions [N, T_k, key_dim] + where N is the batch size, T_k is the sequence length of keys, + and key_dim is the dimensionality of a single key vector. + + Returns: + torch.Tensor: The output tensor of shape [N, T_q, num_units] which + represents the results of the multi-head attention mechanism applied + on the provided queries and keys. + """ + querys = self.W_query(query) # [N, T_q, num_units] + keys = self.W_key(key) # [N, T_k, num_units] + values = self.W_value(key) + split_size = self.num_units // self.num_heads + + querys = torch.stack( + torch.split(querys, split_size, dim=2), dim=0, + ) # [h, N, T_q, num_units/h] + keys = torch.stack( + torch.split(keys, split_size, dim=2), dim=0, + ) # [h, N, T_k, num_units/h] + values = torch.stack( + torch.split(values, split_size, dim=2), dim=0, + ) # [h, N, T_k, num_units/h] + # score = softmax(QK^T / (d_k ** 0.5)) + + scores = torch.matmul(querys, keys.transpose(2, 3)) # [h, N, T_q, T_k] + scores = scores / (self.key_dim**0.5) + scores = F.softmax(scores, dim=3) + # out = score * V + out = torch.matmul(scores, values) # [h, N, T_q, num_units/h] + return torch.cat(torch.split(out, 1, dim=0), dim=3).squeeze( + 0, + ) # [N, T_q, num_units] diff --git a/models/tts/delightful_tts/attention/relative_multi_head_attention.py b/models/tts/delightful_tts/attention/relative_multi_head_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecfe53b7ef46bc1f60ddc1b3540a75edd7eb7cd --- /dev/null +++ b/models/tts/delightful_tts/attention/relative_multi_head_attention.py @@ -0,0 +1,132 @@ +import math +from typing import Tuple + +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F + + +class RelativeMultiHeadAttention(Module): + r"""Multi-head attention with relative positional encoding. + This concept was proposed in the + [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) + + Args: + d_model (int): The dimension of model + num_heads (int): The number of attention heads. + + Inputs: query, key, value, pos_embedding, mask + - **query** (batch, time, dim): Tensor containing query vector + - **key** (batch, time, dim): Tensor containing key vector + - **value** (batch, time, dim): Tensor containing value vector + - **pos_embedding** (batch, time, dim): Positional embedding tensor + - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked + Returns: + - **outputs**: Tensor produces by relative multi head attention module. + + Note: `d_model` should be divisible by `num_heads` in other words `d_model % num_heads` should be zero. + """ + + def __init__( + self, + d_model: int = 512, + num_heads: int = 16, + ): + super().__init__() + assert d_model % num_heads == 0, "d_model % num_heads should be zero." + self.d_model = d_model + self.d_head = int(d_model / num_heads) + self.num_heads = num_heads + self.sqrt_dim = math.sqrt(d_model) + + self.query_proj = nn.Linear(d_model, d_model) + self.key_proj = nn.Linear(d_model, d_model, bias=False) + self.value_proj = nn.Linear(d_model, d_model, bias=False) + self.pos_proj = nn.Linear(d_model, d_model, bias=False) + + self.u_bias = nn.Parameter(torch.Tensor(self.num_heads, self.d_head)) + self.v_bias = nn.Parameter(torch.Tensor(self.num_heads, self.d_head)) + + torch.nn.init.xavier_uniform_(self.u_bias) + torch.nn.init.xavier_uniform_(self.v_bias) + self.out_proj = nn.Linear(d_model, d_model) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + pos_embedding: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Function applies multi-head attention along with relative positional encoding to the inputs. It restructures the input queries, keys, and values according to individual attention heads, applies biases, calculates content and position scores, and combines these to get the final score. A softmax activation is applied over the final score, followed by the calculation of context (contextual representation of input). + + Performs the forward pass on the queries, keys, values, and positional embeddings with a mask. + + Args: + query (torch.Tensor): The input tensor containing query vectors. + key (torch.Tensor): The input tensor containing key vectors. + value (torch.Tensor): The input tensor containing value vectors. + pos_embedding (torch.Tensor): The positional embedding tensor. + mask (torch.Tensor): The mask tensor containing indices to be masked. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The context and attention tensors. + Tensor produces by relative multi head attention module. + """ + batch_size = query.shape[0] + query = self.query_proj(query).view(batch_size, -1, self.num_heads, self.d_head) + key = ( + self.key_proj(key) + .view(batch_size, -1, self.num_heads, self.d_head) + .permute(0, 2, 1, 3) + ) + value = ( + self.value_proj(value) + .view(batch_size, -1, self.num_heads, self.d_head) + .permute(0, 2, 1, 3) + ) + pos_embedding = self.pos_proj(pos_embedding).view( + batch_size, -1, self.num_heads, self.d_head, + ) + u_bias = self.u_bias.expand_as(query) + v_bias = self.v_bias.expand_as(query) + a = (query + u_bias).transpose(1, 2) + content_score = a @ key.transpose(2, 3) + b = (query + v_bias).transpose(1, 2) + pos_score = b @ pos_embedding.permute(0, 2, 3, 1) + pos_score = self._relative_shift(pos_score) + + score = content_score + pos_score + score = score * (1.0 / self.sqrt_dim) + + score.masked_fill_(mask, -1e9) + + attn = F.softmax(score, -1) + + context = (attn @ value).transpose(1, 2) + context = context.contiguous().view(batch_size, -1, self.d_model) + + return self.out_proj(context), attn + + def _relative_shift(self, pos_score: torch.Tensor) -> torch.Tensor: + r"""The main idea of relative positional encoding is that the attention score doesn't only depend on the query and the key, but also on the relative position of the key with respect to the query. This becomes particularly useful when working with sequences of tokens, like in NLP tasks, as it helps the model to be aware of the position of the words (or tokens) in the sentence. + + Performs the relative shift operation on the positional scores. + + Args: + pos_score (torch.Tensor): The positional scores tensor. + + Returns: + torch.Tensor: The shifted positional scores tensor. + """ + batch_size, num_heads, seq_length1, seq_length2 = pos_score.size() + zeros = torch.zeros( + (batch_size, num_heads, seq_length1, 1), device=pos_score.device, + ) + padded_pos_score = torch.cat([zeros, pos_score], dim=-1) + padded_pos_score = padded_pos_score.view( + batch_size, num_heads, seq_length2 + 1, seq_length1, + ) + return padded_pos_score[:, :, 1:].view_as(pos_score) diff --git a/models/tts/delightful_tts/attention/style_embed_attention.py b/models/tts/delightful_tts/attention/style_embed_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..dfc59a5db8986f33ac9045878741ed2add7e64ad --- /dev/null +++ b/models/tts/delightful_tts/attention/style_embed_attention.py @@ -0,0 +1,86 @@ +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F + + +class StyleEmbedAttention(Module): + r"""Mechanism is being used to extract style features from audio data in the form of spectrograms. + + Each style token (parameterized by an embedding vector) represents a unique style feature. The model applies the `StyleEmbedAttention` mechanism to combine these style tokens (style features) in a weighted manner. The output of the attention module is a sum of style tokens, with each token weighted by its relevance to the input. + + This technique is often used in text-to-speech synthesis (TTS) such as Tacotron-2, where the goal is to modulate the prosody, stress, and intonation of the synthesized speech based on the reference audio or some control parameters. The concept of "global style tokens" (GST) was introduced in + [Style Tokens: Unsupervised Style Modeling, Control and Transfer in End-to-End Speech Synthesis](https://arxiv.org/abs/1803.09017) by Yuxuan Wang et al. + + The `StyleEmbedAttention` class is a PyTorch module implementing the attention mechanism. + This class is specifically designed for handling multiple attention heads. + Attention here operates on a query and a set of key-value pairs to produce an output. + + Builds the `StyleEmbedAttention` network. + + Args: + query_dim (int): Dimensionality of the query vectors. + key_dim (int): Dimensionality of the key vectors. + num_units (int): Total dimensionality of the query, key, and value vectors. + num_heads (int): Number of parallel attention layers (heads). + + Note: `num_units` should be divisible by `num_heads`. + """ + + def __init__( + self, + query_dim: int, + key_dim: int, + num_units: int, + num_heads: int, + ): + super().__init__() + self.num_units = num_units + self.num_heads = num_heads + self.key_dim = key_dim + + self.W_query = nn.Linear( + in_features=query_dim, + out_features=num_units, + bias=False, + ) + self.W_key = nn.Linear(in_features=key_dim, out_features=num_units, bias=False) + self.W_value = nn.Linear( + in_features=key_dim, out_features=num_units, bias=False, + ) + + def forward(self, query: torch.Tensor, key_soft: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the StyleEmbedAttention module calculates the attention scores. + + Args: + query (torch.Tensor): The input tensor for queries of shape `[N, T_q, query_dim]` + key_soft (torch.Tensor): The input tensor for keys of shape `[N, T_k, key_dim]` + + Returns: + out (torch.Tensor): The output tensor of shape `[N, T_q, num_units]` + """ + values = self.W_value(key_soft) + split_size = self.num_units // self.num_heads + values = torch.stack(torch.split(values, split_size, dim=2), dim=0) + + # out_soft = scores_soft = None + queries = self.W_query(query) # [N, T_q, num_units] + keys = self.W_key(key_soft) # [N, T_k, num_units] + + # [h, N, T_q, num_units/h] + queries = torch.stack(torch.split(queries, split_size, dim=2), dim=0) + # [h, N, T_k, num_units/h] + keys = torch.stack(torch.split(keys, split_size, dim=2), dim=0) + # [h, N, T_k, num_units/h] + + # score = softmax(QK^T / (d_k ** 0.5)) + scores_soft = torch.matmul(queries, keys.transpose(2, 3)) # [h, N, T_q, T_k] + scores_soft = scores_soft / (self.key_dim**0.5) + scores_soft = F.softmax(scores_soft, dim=3) + + # out = score * V + # [h, N, T_q, num_units/h] + out_soft = torch.matmul(scores_soft, values) + return torch.cat(torch.split(out_soft, 1, dim=0), dim=3).squeeze( + 0, + ) # [N, T_q, num_units] scores_soft diff --git a/models/tts/delightful_tts/attention/tests/__init__.py b/models/tts/delightful_tts/attention/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/tts/delightful_tts/attention/tests/test_conformer.py b/models/tts/delightful_tts/attention/tests/test_conformer.py new file mode 100644 index 0000000000000000000000000000000000000000..c6e0103d438d607dd861d0e84e883bd089b4ec89 --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_conformer.py @@ -0,0 +1,114 @@ +import unittest + +import torch + +from models.config import ( + AcousticENModelConfig, + AcousticPretrainingConfig, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.helpers.initializer import ( + init_acoustic_model, + init_conformer, + init_forward_trains_params, + init_mask_input_embeddings_encoding_attn_mask, +) +from models.tts.delightful_tts.attention.conformer import Conformer + + +# Conformer is used in the encoder of the AccousticModel, crucial for the training +# Here you can understand the input and output shapes of the Conformer +# Integration test +class TestConformer(unittest.TestCase): + def setUp(self): + self.acoustic_pretraining_config = AcousticPretrainingConfig() + self.model_config = AcousticENModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + # Based on speaker.json mock + n_speakers = 10 + + # # Add Conformer as encoder + self.encoder, _ = init_conformer(self.model_config) + + # Add AcousticModel instance + self.acoustic_model, _ = init_acoustic_model( + self.preprocess_config, + self.model_config, + n_speakers, + ) + + # Generate mock data for the forward pass + self.forward_train_params = init_forward_trains_params( + self.model_config, + self.acoustic_pretraining_config, + self.preprocess_config, + n_speakers, + ) + + def test_initialization(self): + """Test that a Conformer instance is correctly initialized.""" + self.assertIsInstance(self.encoder, Conformer) + + def test_forward(self): + """Test that a Conformer instance can correctly perform a forward pass. + For this test case we use the code from AccousticModel. + """ + ( + src_mask, + x, + embeddings, + encoding, + _, + ) = init_mask_input_embeddings_encoding_attn_mask( + self.acoustic_model, + self.forward_train_params, + self.model_config, + ) + + # Assert the shape of x + self.assertEqual( + x.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.speaker_embed_dim // 2, + ], + ), + ) + + # Assert the shape of embeddings + self.assertEqual( + embeddings.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.speaker_embed_dim + + self.model_config.lang_embed_dim, + ], + ), + ) + + # Run conformer encoder + # x: Tensor containing the encoded sequences. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim] + x = self.encoder(x, src_mask, embeddings=embeddings, encoding=encoding) + + # Assert the shape of x + self.assertEqual( + x.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.speaker_embed_dim // 2, + ], + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_conformer_block.py b/models/tts/delightful_tts/attention/tests/test_conformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..ea7554fbd7d627bccfa20454ee769fa0a4801325 --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_conformer_block.py @@ -0,0 +1,126 @@ +import unittest + +import torch + +from models.config import ( + AcousticENModelConfig, + AcousticPretrainingConfig, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.helpers.initializer import ( + init_acoustic_model, + init_conformer, + init_forward_trains_params, + init_mask_input_embeddings_encoding_attn_mask, +) +from models.tts.delightful_tts.attention.conformer_block import ConformerBlock + + +# ConformerBlock is used in the Conformer, crucial for the training +# Here you can understand the input and output shapes of the Conformer +# Integration test +class TestConformerBlock(unittest.TestCase): + def setUp(self): + self.acoustic_pretraining_config = AcousticPretrainingConfig() + self.model_config = AcousticENModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + # Based on speaker.json mock + n_speakers = 10 + + # Init conformer config + _, self.conformer_config = init_conformer( + self.model_config, + ) + + # Add AcousticModel instance + self.acoustic_model, _ = init_acoustic_model( + self.preprocess_config, + self.model_config, + n_speakers, + ) + + self.model = ConformerBlock( + d_model=self.conformer_config.dim, + n_head=self.conformer_config.n_heads, + kernel_size_conv_mod=self.conformer_config.kernel_size_conv_mod, + embedding_dim=self.conformer_config.embedding_dim, + dropout=self.conformer_config.p_dropout, + with_ff=self.conformer_config.with_ff, + ) + + # Generate mock data for the forward pass + self.forward_train_params = init_forward_trains_params( + self.model_config, + self.acoustic_pretraining_config, + self.preprocess_config, + n_speakers, + ) + + def test_initialization(self): + """Test for successful creation of ConformerBlock instance.""" + self.assertIsInstance(self.model, ConformerBlock) + + def test_forward(self): + """Test for successful forward pass.""" + ( + src_mask, + x, + embeddings, + encoding, + attn_mask, + ) = init_mask_input_embeddings_encoding_attn_mask( + self.acoustic_model, + self.forward_train_params, + self.model_config, + ) + + # Run conformer block model + # x: Tensor containing the encoded sequences. Shape: [speaker_embed_dim, batch_size, speaker_embed_dim] + x = self.model( + x, + mask=src_mask, + slf_attn_mask=attn_mask, + embeddings=embeddings, + encoding=encoding, + ) + + # Assert the shape of x + self.assertEqual( + x.shape, + torch.Size( + [ + self.model_config.speaker_embed_dim, + self.acoustic_pretraining_config.batch_size, + self.model_config.speaker_embed_dim // 2, + ], + ), + ) + + def test_with_ff_flag(self): + """Test for correct response based on `with_ff` flag during initialization.""" + model = ConformerBlock( + d_model=20, + n_head=5, + kernel_size_conv_mod=5, + embedding_dim=20, + dropout=0.4, + with_ff=False, + ) + self.assertFalse(hasattr(model, "ff")) + + model = ConformerBlock( + d_model=12, + n_head=6, + kernel_size_conv_mod=3, + embedding_dim=12, + dropout=0.1, + with_ff=True, + ) + self.assertTrue(hasattr(model, "ff")) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_conformer_conv_module.py b/models/tts/delightful_tts/attention/tests/test_conformer_conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..c3c98966f26b015d0cdc0780c6892afd63f89a24 --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_conformer_conv_module.py @@ -0,0 +1,34 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention.conformer_conv_module import ( + ConformerConvModule, +) + + +class TestConformerConvModule(unittest.TestCase): + def test_forward_output_shape(self): + """Test that the output shape from the forward method matches the expected shape.""" + d_model = 10 + kernel_size = 3 + dropout = 0.2 + + model = ConformerConvModule(d_model, kernel_size=kernel_size, dropout=dropout) + + batch_size = 5 + seq_len = 7 + num_features = d_model + + # Create a random tensor to act as the input + x = torch.randn(batch_size, seq_len, num_features) + + # Forward pass through the model + output = model(x) + + # Check the output has the expected shape (batch_size, seq_len, num_features) + self.assertEqual(output.shape, (batch_size, seq_len, num_features)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_conformer_multi_headed_self_attention.py b/models/tts/delightful_tts/attention/tests/test_conformer_multi_headed_self_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..50c7984b85ef335c0a3984067633c66b8520bef4 --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_conformer_multi_headed_self_attention.py @@ -0,0 +1,56 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention import ConformerMultiHeadedSelfAttention + + +# Test class for the ConformerMultiHeadedSelfAttention class +class TestConformerMultiHeadedSelfAttention(unittest.TestCase): + def test_forward(self): + # Create an instance of ConformerMultiHeadedSelfAttention + model = ConformerMultiHeadedSelfAttention( + 512, 2, 0.1, + ) # 512 dim, 2 heads, 10% dropout + + # Generate some random data for input + batch_size = 2 + seq_length = 15 + query = torch.rand( + batch_size, + seq_length, + 512, + ) + key = torch.rand( + batch_size, + seq_length, + 512, + ) + value = torch.rand( + batch_size, + seq_length, + 512, + ) + mask = torch.ones( + batch_size, + 1, + seq_length, + dtype=torch.bool, + ) + encoding = torch.rand( + 1, + seq_length, + 512, + ) + + # Execute the forward pass + outputs, attn = model(query, key, value, mask, encoding) + + # Check the output shapes + self.assertEqual(outputs.shape, (batch_size, seq_length, 512)) + self.assertEqual(attn.shape, (batch_size, 2, seq_length, seq_length)) + + +# Run the tests +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_feed_forward.py b/models/tts/delightful_tts/attention/tests/test_feed_forward.py new file mode 100644 index 0000000000000000000000000000000000000000..3a63af3789f34fd179325a8ffd18215a5385c7de --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_feed_forward.py @@ -0,0 +1,39 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention.feed_forward import FeedForward + + +class TestFeedForward(unittest.TestCase): + def setUp(self): + self.d_model = 10 + self.kernel_size = 3 + self.dropout = 0.2 + self.expansion_factor = 4 + + self.model = FeedForward( + self.d_model, + self.kernel_size, + self.dropout, + self.expansion_factor, + ) + + def test_forward(self): + batch_size = 5 + seq_len = 7 + num_features = self.d_model + + # Create a random tensor to act as the input + x = torch.randn(batch_size, seq_len, num_features) + + # Forward pass + output = self.model(x) + + # Check the resultant tensor after forward pass + self.assertIsInstance(output, torch.Tensor) + self.assertEqual(output.shape, (batch_size, seq_len, num_features)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_multi_head_attention.py b/models/tts/delightful_tts/attention/tests/test_multi_head_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..c790675659c9da1ecb779033db4608752c520fcb --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_multi_head_attention.py @@ -0,0 +1,44 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention.multi_head_attention import MultiHeadAttention + + +class TestMultiHeadAttention(unittest.TestCase): + def setUp(self): + # Initialize an instance of MultiHeadAttention + self.attention = MultiHeadAttention( + query_dim=512, key_dim=512, num_units=512, num_heads=8, + ) + # Assuming batch=3, seq_length=10, dim=512 + self.dim_params = (3, 10, 512) + self.query = torch.rand(self.dim_params) # [N, T_q, query_dim] + self.key = torch.rand(self.dim_params) # [N, T_k, key_dim] + + def test_forward(self): + # Test the forward function + out = self.attention(self.query, self.key) + + # Assert output shape + self.assertEqual(out.shape, self.dim_params) + + def test_dtype(self): + # Test forward function + out = self.attention(self.query, self.key) + + # Check the data type of output + self.assertTrue(out.dtype == torch.float32) + + def test_consistent_output(self): + # Test forward function + out1 = self.attention(self.query, self.key) + out2 = self.attention(self.query, self.key) + + # Check if the output is consistent for the same input + self.assertTrue(torch.allclose(out1, out2)) + + +# Run the tests +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_relative_multi_head_attention.py b/models/tts/delightful_tts/attention/tests/test_relative_multi_head_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..f334820e2ab67675a460bb79b9eb445d4f055dbc --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_relative_multi_head_attention.py @@ -0,0 +1,70 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention.relative_multi_head_attention import ( + RelativeMultiHeadAttention, +) + + +class TestRelativeMultiHeadAttention(unittest.TestCase): + def setUp(self): + # Initialize an instance of RelativeMultiHeadAttention + self.attention = RelativeMultiHeadAttention(d_model=512, num_heads=8) + + # Generate random tensors for query, key, value, pos_embedding, mask + # Assuming batch=3, seq_length=10, dim=512 + self.params_shape = (3, 10, 512) + self.query = torch.rand(self.params_shape) + self.key = torch.rand(self.params_shape) + self.value = torch.rand(self.params_shape) + self.pos_embedding = torch.rand(self.params_shape) + + # A simple test case without actual masked positions + self.mask_shape = (3, 8, 10, 10) + self.mask = torch.zeros(self.mask_shape).type(torch.bool) + + def test_init_assert(self): + # Test initializing with an invalid d_model and num_head pair + with self.assertRaises(AssertionError): + RelativeMultiHeadAttention(d_model=512, num_heads=7) + + def test_forward(self): + # Generate random tensors for query, key, value, pos_embedding, mask + # Test the forward function + context, attn = self.attention( + self.query, self.key, self.value, self.pos_embedding, self.mask, + ) + + # Assert output shapes + self.assertEqual(context.shape, self.params_shape) + self.assertEqual(attn.shape, self.mask_shape) + # Check data types of outputs + self.assertTrue(context.dtype == torch.float32) + self.assertTrue(attn.dtype == torch.float32) + + def test_relative_shift(self): + # Generate a random positional score tensor + # Assuming batch=3, num_heads=8, seq_length1=10, seq_length2=10 + params_shape = (*self.params_shape, 10) + pos_score = torch.rand(params_shape) + + # Test the _relative_shift function + shifted_pos_score = self.attention._relative_shift(pos_score) + + # Assert output shape and values + self.assertEqual(shifted_pos_score.shape, params_shape) + self.assertTrue(torch.all((shifted_pos_score >= 0) & (shifted_pos_score <= 1))) + + def test_attention_values(self): + # Test the forward function + context, attn = self.attention( + self.query, self.key, self.value, self.pos_embedding, self.mask, + ) + + # Check values of attention output are in range [0, 1] + self.assertTrue(torch.all((attn >= 0) & (attn <= 1))) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/attention/tests/test_style_embed_attention.py b/models/tts/delightful_tts/attention/tests/test_style_embed_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..267608f519c8f6f56cbe1813ef43eea5f75f3dd6 --- /dev/null +++ b/models/tts/delightful_tts/attention/tests/test_style_embed_attention.py @@ -0,0 +1,39 @@ +import unittest + +import torch + +from models.tts.delightful_tts.attention.style_embed_attention import ( + StyleEmbedAttention, +) + + +class TestStyleEmbedAttention(unittest.TestCase): + def test_output_shape_and_value_range(self): + model = StyleEmbedAttention( + query_dim=16, + key_dim=16, + num_units=32, + num_heads=4, + ) + query = torch.rand( + 5, + 10, + 16, + ) # batch of 5, 10 queries per batch, each of size 16 + key = torch.rand( + 5, + 20, + 16, + ) # batch of 5, 20 key-value pairs per batch, each of size 16 + output = model(query, key) + + # Check that output shape is as expected + self.assertEqual(output.shape, (5, 10, 32)) + + # Check that the values in the output tensor are within a valid range after softmax and matmul (i.e., [-1, 1]). + self.assertTrue(torch.all((output > -1) & (output < 1))) + + +# Run the tests +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/constants.py b/models/tts/delightful_tts/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..edd7f40e34518191891f23ff7bd1f26897976e36 --- /dev/null +++ b/models/tts/delightful_tts/constants.py @@ -0,0 +1,4 @@ +# config.py + +# Setting LeakyReLU activation function parameter +LEAKY_RELU_SLOPE = 0.3 diff --git a/models/tts/delightful_tts/conv_blocks/__init__.py b/models/tts/delightful_tts/conv_blocks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..40b160e41d16f13e5a6e157adf4dce2e3a31e711 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/__init__.py @@ -0,0 +1,7 @@ +from .activation import GLUActivation +from .add_coords import AddCoords +from .bsconv import BSConv1d +from .conv1d import DepthWiseConv1d, PointwiseConv1d +from .conv1d_glu import Conv1dGLU +from .conv_transposed import ConvTransposed +from .coord_conv1d import CoordConv1d diff --git a/models/tts/delightful_tts/conv_blocks/activation.py b/models/tts/delightful_tts/conv_blocks/activation.py new file mode 100644 index 0000000000000000000000000000000000000000..3ed863d0bf7f036a19b65768f23e2b2a47de6bf8 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/activation.py @@ -0,0 +1,54 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE + + +class GLUActivation(Module): + r"""Implements the Gated Linear Unit (GLU) activation function. + + The GLU activation splits the input in half across the channel dimension. + One half is passed through a nonlinear activation function (like sigmoid or leaky ReLU), + and the output from this activation function is used as a gate to control the + amplitude of the other half of the input. An element-wise multiplication is then performed + between the gating signal and the other half of the input. + + The GLU activation allows the model to dynamically choose which inputs to pass through and + what information to suppress, which can help improving the model performance on certain tasks. + + Args: + slope: Controls the slope for the leaky ReLU activation function. Default: 0.3 or see the const `LEAKY_RELU_SLOPE` + + Shape: + - Input: (N, 2*C, L) where C is the number of input channels. + - Output: (N, C, L) + + Examples: + ```python + m = GLUActivation(0.3) + input = torch.randn(16, 2*20, 44) + output = m(input) + ``` + + """ + + def __init__(self, slope: float = LEAKY_RELU_SLOPE): + super().__init__() + self.lrelu = nn.LeakyReLU(slope) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Defines the computation performed at every call. + + Args: + x: The input tensor of shape (batch_size, 2*channels, signal_length) + + Returns: + x: The output tensor of shape (batch_size, channels, signal_length) + """ + # Split the input into two equal parts (chunks) along dimension 1 + out, gate = x.chunk(2, dim=1) + + # Perform element-wise multiplication of the first half (out) + # with the result of applying LeakyReLU on the second half (gate) + return out * self.lrelu(gate) diff --git a/models/tts/delightful_tts/conv_blocks/add_coords.py b/models/tts/delightful_tts/conv_blocks/add_coords.py new file mode 100644 index 0000000000000000000000000000000000000000..22d1debd073c153752ddb5ae3d8a648f02a8833f --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/add_coords.py @@ -0,0 +1,133 @@ +import torch +from torch.nn import Module + + +class AddCoords(Module): + r"""AddCoords is a PyTorch module that adds additional channels to the input tensor containing the relative + (normalized to `[-1, 1]`) coordinates of each input element along the specified number of dimensions (`rank`). + Essentially, it adds spatial context information to the tensor. + + Typically, these inputs are feature maps coming from some CNN, where the spatial organization of the input + matters (such as an image or speech signal). + + This additional spatial context allows subsequent layers (such as convolutions) to learn position-dependent + features. For example, in tasks where the absolute position of features matters (such as denoising and + segmentation tasks), it helps the model to know where (in terms of relative position) the features are. + + Args: + rank (int): The dimensionality of the input tensor. That is to say, this tells us how many dimensions the + input tensor's spatial context has. It's assumed to be 1, 2, or 3 corresponding to some 1D, 2D, + or 3D data (like an image). + + with_r (bool): Boolean indicating whether to add an extra radial distance channel or not. If True, an extra + channel is appended, which measures the Euclidean (L2) distance from the center of the image. + This might be useful when the proximity to the center of the image is important to the task. + """ + + def __init__(self, rank: int, with_r: bool = False): + super().__init__() + self.rank = rank + self.with_r = with_r + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the AddCoords module. Depending on the rank of the tensor, it adds one or more new channels + with relative coordinate values. If `with_r` is True, an extra radial channel is included. + + For example, for an image (`rank=2`), two channels would be added which contain the normalized x and y + coordinates respectively of each pixel. + + Calling the forward method updates the original tensor `x` with the added channels. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + out (torch.Tensor): The input tensor with added coordinate and possibly radial channels. + """ + if self.rank == 1: + batch_size_shape, _, dim_x = x.shape + xx_range = torch.arange(dim_x, dtype=torch.int32, device=x.device) + xx_channel = xx_range[None, None, :] + + xx_channel = xx_channel.float() / (dim_x - 1) + xx_channel = xx_channel * 2 - 1 + xx_channel = xx_channel.repeat(batch_size_shape, 1, 1) + + out = torch.cat([x, xx_channel], dim=1) + + if self.with_r: + rr = torch.sqrt(torch.pow(xx_channel - 0.5, 2)) + out = torch.cat([out, rr], dim=1) + + elif self.rank == 2: + batch_size_shape, _, dim_y, dim_x = x.shape + xx_ones = torch.ones([1, 1, 1, dim_x], dtype=torch.int32, device=x.device) + yy_ones = torch.ones([1, 1, 1, dim_y], dtype=torch.int32, device=x.device) + + xx_range = torch.arange(dim_y, dtype=torch.int32, device=x.device) + yy_range = torch.arange(dim_x, dtype=torch.int32, device=x.device) + xx_range = xx_range[None, None, :, None] + yy_range = yy_range[None, None, :, None] + + xx_channel = torch.matmul(xx_range, xx_ones) + yy_channel = torch.matmul(yy_range, yy_ones) + + # transpose y + yy_channel = yy_channel.permute(0, 1, 3, 2) + + xx_channel = xx_channel.float() / (dim_y - 1) + yy_channel = yy_channel.float() / (dim_x - 1) + + xx_channel = xx_channel * 2 - 1 + yy_channel = yy_channel * 2 - 1 + + xx_channel = xx_channel.repeat(batch_size_shape, 1, 1, 1) + yy_channel = yy_channel.repeat(batch_size_shape, 1, 1, 1) + + out = torch.cat([x, xx_channel, yy_channel], dim=1) + + if self.with_r: + rr = torch.sqrt( + torch.pow(xx_channel - 0.5, 2) + torch.pow(yy_channel - 0.5, 2), + ) + out = torch.cat([out, rr], dim=1) + + elif self.rank == 3: + batch_size_shape, _, dim_z, dim_y, dim_x = x.shape + xx_ones = torch.ones([1, 1, 1, 1, dim_x], dtype=torch.int32, device=x.device) + yy_ones = torch.ones([1, 1, 1, 1, dim_y], dtype=torch.int32, device=x.device) + zz_ones = torch.ones([1, 1, 1, 1, dim_z], dtype=torch.int32, device=x.device) + + xy_range = torch.arange(dim_y, dtype=torch.int32, device=x.device) + xy_range = xy_range[None, None, None, :, None] + + yz_range = torch.arange(dim_z, dtype=torch.int32, device=x.device) + yz_range = yz_range[None, None, None, :, None] + + zx_range = torch.arange(dim_x, dtype=torch.int32, device=x.device) + zx_range = zx_range[None, None, None, :, None] + + xy_channel = torch.matmul(xy_range, xx_ones) + xx_channel = torch.cat([xy_channel + i for i in range(dim_z)], dim=2) + + yz_channel = torch.matmul(yz_range, yy_ones) + yz_channel = yz_channel.permute(0, 1, 3, 4, 2) + yy_channel = torch.cat([yz_channel + i for i in range(dim_x)], dim=4) + + zx_channel = torch.matmul(zx_range, zz_ones) + zx_channel = zx_channel.permute(0, 1, 4, 2, 3) + zz_channel = torch.cat([zx_channel + i for i in range(dim_y)], dim=3) + + out = torch.cat([x, xx_channel, yy_channel, zz_channel], dim=1) + + if self.with_r: + rr = torch.sqrt( + torch.pow(xx_channel - 0.5, 2) + + torch.pow(yy_channel - 0.5, 2) + + torch.pow(zz_channel - 0.5, 2), + ) + out = torch.cat([out, rr], dim=1) + else: + raise NotImplementedError + + return out diff --git a/models/tts/delightful_tts/conv_blocks/bsconv.py b/models/tts/delightful_tts/conv_blocks/bsconv.py new file mode 100644 index 0000000000000000000000000000000000000000..5929e035a7b248fe2f6650d3d7c019172795583f --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/bsconv.py @@ -0,0 +1,57 @@ +import torch +from torch.nn import Module + +from .conv1d import DepthWiseConv1d, PointwiseConv1d + + +class BSConv1d(Module): + r"""`BSConv1d` implements the `BSConv` concept which is based on the paper [BSConv: + Binarized Separated Convolutional Neural Networks](https://arxiv.org/pdf/2003.13549.pdf). + + `BSConv` is an amalgamation of depthwise separable convolution and pointwise convolution. + Depthwise separable convolution utilizes far fewer parameters by separating the spatial + (depthwise) and channel-wise (pointwise) operations. Meanwhile, pointwise convolution + helps in transforming the channel characteristics without considering the channel's context. + + Args: + channels_in (int): Number of input channels + channels_out (int): Number of output channels produced by the convolution + kernel_size (int): Size of the kernel used in depthwise convolution + padding (int): Zeropadding added around the input tensor along the height and width directions + + Attributes: + pointwise (PointwiseConv1d): Pointwise convolution module + depthwise (DepthWiseConv1d): Depthwise separable convolution module + """ + + def __init__( + self, + channels_in: int, + channels_out: int, + kernel_size: int, + padding: int, + ): + super().__init__() + + # Instantiate Pointwise Convolution Module: + # First operation in BSConv: the number of input channels is transformed to the number + # of output channels without taking into account the channel context. + self.pointwise = PointwiseConv1d(channels_in, channels_out) + + # Instantiate Depthwise Convolution Module: + # Second operation in BSConv: A spatial convolution is performed independently over each output + # channel from the pointwise convolution. + self.depthwise = DepthWiseConv1d( + channels_out, + channels_out, + kernel_size=kernel_size, + padding=padding, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Propagate input tensor through pointwise convolution. + x1 = self.pointwise(x) + + # Propagate the result of the previous pointwise convolution through the depthwise convolution. + # Return final output of the sequence of pointwise and depthwise convolutions + return self.depthwise(x1) diff --git a/models/tts/delightful_tts/conv_blocks/conv1d.py b/models/tts/delightful_tts/conv_blocks/conv1d.py new file mode 100644 index 0000000000000000000000000000000000000000..f77926b44c5f3b7335b1dbb42e462e6cba27ad11 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/conv1d.py @@ -0,0 +1,171 @@ +import torch +from torch import nn +from torch.nn import Module + + +class DepthWiseConv1d(Module): + r"""Implements Depthwise 1D convolution. This module will apply a spatial convolution over inputs + independently over each input channel in the style of depthwise convolutions. + + In a depthwise convolution, each input channel is convolved with its own set of filters, as opposed + to standard convolutions where each input channel is convolved with all filters. + At `groups=in_channels`, each input channel is convolved with its own set of filters. + Filters in the + DepthwiseConv1d are not shared among channels. This method can drastically reduce the number of + parameters/learnable weights in the model, as each input channel gets its own filter. + + This technique is best suited to scenarios where the correlation between different channels is + believed to be low. It is commonly employed in MobileNet models due to the reduced number of + parameters, which is critical in mobile devices where computational resources are limited. + + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + kernel_size (int): Size of the convolving kernel + padding (int): Zero-padding added to both sides of the input + + Shape: + - Input: (N, C_in, L_in) + - Output: (N, C_out, L_out), where + + `L_out = [L_in + 2*padding - (dilation*(kernel_size-1) + 1)]/stride + 1` + + Attributes: + weight (Tensor): the learnable weights of shape (`out_channels`, `in_channels`/`group`, `kernel_size`) + bias (Tensor, optional): the learnable bias of the module of shape (`out_channels`) + + Examples: + ```python + m = DepthWiseConv1d(16, 33, 3, padding=1) + input = torch.randn(20, 16, 50) + output = m(input) + ``` + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + padding: int, + ): + super().__init__() + + self.conv = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + padding=padding, + groups=in_channels, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Defines the computation performed at every call. + + Args: + x: input tensor of shape (batch_size, in_channels, signal_length) + + Returns: + output tensor of shape (batch_size, out_channels, signal_length) + """ + return self.conv(x) + + +class PointwiseConv1d(Module): + r"""Applies a 1D pointwise (aka 1x1) convolution over an input signal composed of several input + planes, officially known as channels in this context. + + The operation implemented is also known as a "channel mixing" operation, as each output channel can be + seen as a linear combination of input channels. + + In the simplest case, the output value of the layer with input size + (N, C_in, L) and output (N, C_out, L_out) can be + precisely described as: + + $$out(N_i, C_{out_j}) = bias(C_{out_j}) + + weight(C_{out_j}, k) * input(N_i, k)$$ + + where 'N' is a batch size, 'C' denotes a number of channels, + 'L' is a length of signal sequence. + The symbol '*' in the above indicates a 1D cross-correlation operation. + + The 1D cross correlation operation "*": [Wikipedia Cross-correlation](https://en.wikipedia.org/wiki/Cross-correlation) + + This module supports `TensorFloat32`. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + stride (int): Stride of the convolution. Default: 1 + padding (int): Zero-padding added to both sides of the input. Default: 0 + bias (bool): If set to False, the layer will not learn an additive bias. Default: True + kernel_size (int): Size of the convolving kernel. Default: 1 + + Shape: + - Input: (N, C_in, L_in) + - Output: (N, C_out, L_out), where + + L_out = [L_in + 2*padding - (dilation*(kernel_size-1) + 1)]/stride + 1 + + Attributes: + weight (Tensor): the learnable weights of shape (out_channels, in_channels, kernel_size) + bias (Tensor, optional): the learnable bias of the module of shape (out_channels) + + Example: + ```python + m = PointwiseConv1d(16, 33, 1, padding=0, bias=True) + input = torch.randn(20, 16, 50) + output = m(input) + ``` + + + Description of parameters: + stride (default 1): Controls the stride for the operation, which is the number of steps the convolutional + kernel moves for each operation. A stride of 1 means that the kernel moves one step at a time and a stride + of 2 means skipping every other step. Higher stride values can down sample the output and lead to smaller + output shapes. + + padding (default 0): Controls the amount of padding applied to the input. By adding padding, the spatial + size of the output can be controlled. If it is set to 0, no padding is applied. If it is set to 1, zero + padding of one pixel width is added to the input data. + + bias (default True): Controls whether the layer uses a bias vector. By default, it is True, meaning that + the layer has a learnable bias parameter. + + kernel_size (default 1): The size of the convolving kernel. In the case of 1D convolution, kernel_size is + a single integer that specifies the number of elements the filter that convolves the input should have. + In your PointwiseConv1d case, the default kernel size is 1, indicating a 1x1 convolution is applied + which is commonly known as a pointwise convolution. + + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + padding: int = 0, + bias: bool = True, + kernel_size: int = 1, + ): + super().__init__() + + self.conv = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Defines the computation performed at every call. + + Args: + x (torch.Tensor): input tensor of shape (batch_size, in_channels, signal_length) + + Returns: + output (torch.Tensor): tensor of shape (batch_size, out_channels, signal_length) + """ + return self.conv(x) diff --git a/models/tts/delightful_tts/conv_blocks/conv1d_glu.py b/models/tts/delightful_tts/conv_blocks/conv1d_glu.py new file mode 100644 index 0000000000000000000000000000000000000000..8a508ffe6fb333546e6c96024c1692bacec45596 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/conv1d_glu.py @@ -0,0 +1,71 @@ +import torch +from torch import nn +from torch.nn import Module + +from .bsconv import BSConv1d + + +class Conv1dGLU(Module): + r"""`Conv1dGLU` implements a variant of Convolutional Layer with a Gated Linear Unit (GLU). + It's based on the Deep Voice 3 project. + + Args: + d_model (int): model dimension parameter. + kernel_size (int): kernel size for the convolution layer. + padding (int): padding size for the convolution layer. + embedding_dim (int): dimension of the embedding. + + Attributes: + bsconv1d (BSConv1d) : an instance of the Binarized Separated Convolution (1d) + embedding_proj (torch.nn.Modules.Linear): linear transformation for embeddings. + sqrt (torch.Tensor): buffer that stores the square root of 0.5 + softsign (torch.nn.SoftSign): SoftSign Activation function + """ + + def __init__( + self, + d_model: int, + kernel_size: int, + padding: int, + embedding_dim: int, + ): + super().__init__() + + self.bsconv1d = BSConv1d( + d_model, + 2 * d_model, + kernel_size=kernel_size, + padding=padding, + ) + + self.embedding_proj = nn.Linear( + embedding_dim, + d_model, + ) + + self.register_buffer("sqrt", torch.sqrt(torch.tensor([0.5])).squeeze(0)) + + self.softsign = torch.nn.Softsign() + + def forward(self, x: torch.Tensor, embeddings: torch.Tensor) -> torch.Tensor: + """Forward propagation method for the Conv1dGLU layer. + + Args: + x (torch.Tensor): input tensor + embeddings (torch.Tensor): input embeddings + + Returns: + x (torch.Tensor): output tensor after application of Conv1dGLU + """ + x = x.permute((0, 2, 1)) + residual = x + x = self.bsconv1d(x) + splitdim = 1 + a, b = x.split(x.size(splitdim) // 2, dim=splitdim) + embeddings = self.embedding_proj(embeddings) + softsign = self.softsign(embeddings) + a = a + softsign.permute((0, 2, 1)) + x = a * torch.sigmoid(b) + x = x + residual + x = x * self.sqrt + return x.permute((0, 2, 1)) diff --git a/models/tts/delightful_tts/conv_blocks/conv_transposed.py b/models/tts/delightful_tts/conv_blocks/conv_transposed.py new file mode 100644 index 0000000000000000000000000000000000000000..2a712b2f44ac653a122221c428102ee73cd62f60 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/conv_transposed.py @@ -0,0 +1,59 @@ +import torch +from torch.nn import Module + +from .bsconv import BSConv1d + + +class ConvTransposed(Module): + r"""`ConvTransposed` applies a 1D convolution operation, with the main difference that it transposes the + last two dimensions of the input tensor before and after applying the `BSConv1d` convolution operation. + This can be useful in certain architectures where the tensor dimensions are processed in a different order. + + The `ConvTransposed` class performs a `BSConv` operation after transposing the input tensor dimensions. Specifically, it swaps the channels and width dimensions of a tensor, applies the convolution, and then swaps the dimensions back to their original order. The intuition behind swapping dimensions can depend on the specific use case in the larger architecture; typically, it's used when the operation or sequence of operations expected a different arrangement of dimensions. + + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + kernel_size (int): Size of the kernel used in convolution + padding (int): Zero-padding added around the input tensor along the width direction + + Attributes: + conv (BSConv1d): `BSConv1d` module to apply convolution. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 1, + padding: int = 0, + ): + super().__init__() + + # Define BSConv1d convolutional layer + self.conv = BSConv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + padding=padding, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward propagation method for the ConvTransposed layer. + + Args: + x (torch.Tensor): input tensor + + Returns: + x (torch.Tensor): output tensor after application of ConvTransposed + """ + # Transpose the last two dimensions (dimension 1 and 2 here). Now the tensor has shape (N, W, C) + x = x.contiguous().transpose(1, 2) + + # Apply BSConv1d convolution. + x = self.conv(x) + + # Transpose the last two dimensions back to their original order. Now the tensor has shape (N, C, W) + # Return final output tensor + return x.contiguous().transpose(1, 2) + diff --git a/models/tts/delightful_tts/conv_blocks/coord_conv1d.py b/models/tts/delightful_tts/conv_blocks/coord_conv1d.py new file mode 100644 index 0000000000000000000000000000000000000000..2af3f437abf501e1d3ab1673b180020eeb4f36a5 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/coord_conv1d.py @@ -0,0 +1,100 @@ +import torch +from torch import nn +from torch.nn import Module +from torch.nn.modules import conv + +from .add_coords import AddCoords + + +class CoordConv1d(conv.Conv1d, Module): + r"""`CoordConv1d` is an extension of the standard 1D convolution layer (`conv.Conv1d`), with the addition of extra coordinate + channels. These extra channels encode positional coordinates, and optionally, the radial distance from the origin. + This is inspired by the paper: + [An Intriguing Failing of Convolutional Neural Networks and the CoordConv Solution](https://arxiv.org/abs/1807.03247) + and is designed to help Convolution layers to pay attention to the absolute position of features in the input space. + + The responsibility of this class is to intercept the input tensor and append extra channels to it. These extra channels + encode the positional coordinates (and optionally, the radial distance from the center). The enhanced tensor is then + immediately passed through a standard Conv1D layer. + + In concrete terms, this means Convolution layer does not just process the color in an image-based task, but also 'knows' + where in the overall image this color is located. + + In a typical Text-To-Speech (TTS) system like DelightfulTTS, the utterance is processed in a sequential manner. + The importance of sequential data in such a use-case can benefit from `CoordConv` layer as it offers a way to draw + more attention to the positioning of data. `CoordConv` is a drop-in replacement for standard convolution layers, + enriches spatial representation in Convolutional Neural Networks (CNN) with additional positional information. + + Hence, the resultant Convolution does not only process the characteristics of the sound in the input speech signal, + but also 'knows' where in the overall signal this particular sound is located, providing it with the spatial context. + This can be particularly useful in TTS systems where the sequence of phonemes and their timing can be critical. + + Args: + in_channels (int): Number of channels in the input. + out_channels (int): Number of channels produced by the convolution. + kernel_size (int): Size of the convolving kernel. + stride (int): Stride of the convolution. Default: 1. + padding (int): Zero-padding added to both sides of the input . Default: 0. + dilation (int): Spacing between kernel elements. Default: 1. + groups (int): Number of blocked connections from input channels to output channels. Default: 1. + bias (bool): If True, adds a learnable bias to the output. Default: True. + with_r (bool): If True, adds a radial coordinate channel. Default: False. + + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + groups: int = 1, + bias: bool = True, + with_r: bool = False, + ): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride, + padding, + dilation, + groups, + bias, + ) + + self.rank = 1 + self.addcoords = AddCoords(self.rank, with_r) + + self.conv = nn.Conv1d( + in_channels + self.rank + int(with_r), + out_channels, + kernel_size, + stride, + padding, + dilation, + groups, + bias, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""The forward pass of the `CoordConv1d` module. It adds the coordinate channels to the input tensor with the `AddCoords` + module, and then immediately passes the result through a 1D convolution. + + As a result, the subsequent Conv layers don't merely process sound characteristics of the speech signal, but are + also aware of their relative positioning, offering a notable improvement over traditional methods, particularly for + challenging TTS tasks where the sequence is critical. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor of shape (batch_size, out_channels, length). + """ + # Apply AddCoords layer to add coordinate channels to the input tensor + x = self.addcoords(x) + + # Apply convolution + return self.conv(x) diff --git a/models/tts/delightful_tts/conv_blocks/tests/__init__.py b/models/tts/delightful_tts/conv_blocks/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_activation.py b/models/tts/delightful_tts/conv_blocks/tests/test_activation.py new file mode 100644 index 0000000000000000000000000000000000000000..19936378d95782ad32f5bb054e60fa8074319c84 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_activation.py @@ -0,0 +1,44 @@ +import unittest + +import torch +from torch.autograd.gradcheck import gradcheck + +from models.tts.delightful_tts.conv_blocks.activation import GLUActivation + + +# Unit Testing Class +class TestGLUActivation(unittest.TestCase): + def setUp(self): + self.glu = GLUActivation() + + # Test that dimensions remain unchanged + def test_dimensions(self): + # random data with shape like (batch_size, channels, height, width) + x = torch.randn(32, 4, 64, 64) + x_after_glu = self.glu(x) + expected_shape = ( + x_after_glu.shape[0], + x_after_glu.shape[1] * 2, + x_after_glu.shape[2], + x_after_glu.shape[3], + ) + self.assertEqual(x.shape, expected_shape) + + # Test the gradients + def test_gradcheck(self): + # use double precision for gradcheck + x = torch.randn(2, 2, dtype=torch.float64, requires_grad=True) + self.assertTrue(gradcheck(self.glu, x), "Gradient check failed") + + # Test for specific values + def test_values(self): + x = torch.tensor([[-0.5, -0.5], [0.5, 0.5]], dtype=torch.float32) + x_after_glu = self.glu(x) + expected_values = torch.tensor([[0.075], [0.25]], dtype=torch.float32) + torch.testing.assert_close( + x_after_glu, expected_values, rtol=1e-4, atol=1e-8, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_add_coords.py b/models/tts/delightful_tts/conv_blocks/tests/test_add_coords.py new file mode 100644 index 0000000000000000000000000000000000000000..7686721901b349c1e818348f921d064de088184d --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_add_coords.py @@ -0,0 +1,63 @@ +import unittest + +import torch + +from models.tts.delightful_tts.conv_blocks.add_coords import AddCoords + + +# Test case for the AddCoords class +class TestAddCoords(unittest.TestCase): + """Test case for the AddCoords class""" + + def test_rank_1_without_r(self): + """Test for rank=1, with_r=False""" + layer = AddCoords(rank=1, with_r=False) + x = torch.rand(4, 3, 10) + out = layer(x) + self.assertEqual(list(out.shape), [4, 4, 10]) + + def test_rank_1_with_r(self): + """Test for rank=1, with_r=True""" + layer = AddCoords(rank=1, with_r=True) + x = torch.rand(4, 3, 10) + out = layer(x) + self.assertEqual(list(out.shape), [4, 5, 10]) + + def test_rank_2_without_r(self): + """Test for rank=2, with_r=False""" + layer = AddCoords(rank=2, with_r=False) + x = torch.rand(4, 3, 10, 20) + out = layer(x) + self.assertEqual(list(out.shape), [4, 5, 10, 20]) + + def test_rank_2_with_r(self): + """Test for rank=2, with_r=True""" + layer = AddCoords(rank=2, with_r=True) + x = torch.rand(4, 3, 10, 20) + out = layer(x) + self.assertEqual(list(out.shape), [4, 6, 10, 20]) + + def test_rank_3_without_r(self): + """Test for rank=3, with_r=False""" + layer = AddCoords(rank=3, with_r=False) + x = torch.rand(1, 3, 10, 20, 30) + out = layer(x) + self.assertEqual(list(out.shape), [1, 6, 10, 20, 30]) + + def test_rank_3_with_r(self): + """Test for rank=3, with_r=True""" + layer = AddCoords(rank=3, with_r=True) + x = torch.rand(1, 3, 10, 20, 30) + out = layer(x) + self.assertEqual(list(out.shape), [1, 7, 10, 20, 30]) + + def test_not_implemented(self): + """Test for not implemented rank.""" + layer = AddCoords(rank=4, with_r=False) + x = torch.rand(4, 3, 10, 20, 30, 40) + self.assertRaises(NotImplementedError, layer, x) + + +# Execute the unit test +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_bsconv.py b/models/tts/delightful_tts/conv_blocks/tests/test_bsconv.py new file mode 100644 index 0000000000000000000000000000000000000000..ead21b11e4143f36a1c96d0f3c9878f94f83cc08 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_bsconv.py @@ -0,0 +1,66 @@ +import unittest + +import torch + +from models.tts.delightful_tts.conv_blocks.bsconv import BSConv1d + + +class TestBSConv1d(unittest.TestCase): + def test_given_kernel_size_and_padding(self): + # Batch size, Input channels, output channels + N, C_in, C_out = 16, 4, 8 + + for kernel_size, padding in [(5, 0), (7, 3), (11, 5)]: + bsconv = BSConv1d( + C_in, + C_out, + kernel_size, + padding, + ) + + t_width = 100 + x = torch.randn( + N, + C_in, + t_width, + ) + + out = bsconv(x) + new_t_width = (t_width + 2 * padding - (kernel_size - 1) - 1) + 1 + + self.assertEqual( + out.shape, + (N, C_out, new_t_width), + f"For kernel_size={kernel_size} and padding={padding}, expected output shape: {N, C_out, new_t_width}, but got: {out.shape}", + ) + + def test_with_different_batch_size_and_input_channels(self): + # Output channels, kernel size, padding + C_out, kernel_size, padding = 16, 3, 1 + + for N, C_in in [(32, 8), (64, 16), (128, 32)]: + bsconv = BSConv1d( + C_in, + C_out, + kernel_size, + padding, + ) + + t_width = 100 + x = torch.randn( + N, + C_in, + t_width, + ) + + out = bsconv(x) + + self.assertEqual( + out.shape, + (N, C_out, t_width), + f"For batch_size={N} and input_channels={C_in}, expected output shape: {N, C_out, 100}, but got: {out.shape}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_conv1d.py b/models/tts/delightful_tts/conv_blocks/tests/test_conv1d.py new file mode 100644 index 0000000000000000000000000000000000000000..a7682662088c143fa0f74ee685563d2c08a25e00 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_conv1d.py @@ -0,0 +1,152 @@ +import unittest + +import torch + +from models.tts.delightful_tts.conv_blocks.conv1d import ( + DepthWiseConv1d, + PointwiseConv1d, +) + + +class TestDepthwiseConv1d(unittest.TestCase): + def setUp(self): + # initialize parameters once and reuse them in multiple test cases + self.in_channels, self.out_channels, self.kernel_size, self.padding = 2, 4, 3, 1 + + self.depthwise_conv = DepthWiseConv1d( + self.in_channels, + self.out_channels, + self.kernel_size, + self.padding, + ) + + # Generate an input data + self.x_rand = torch.randn( + 32, + self.in_channels, + 64, + ) + self.x_zero = torch.zeros( + 32, + self.in_channels, + 64, + ) + self.x_ones = torch.ones( + 32, + self.in_channels, + 64, + ) + + def test_forward(self): + out = self.depthwise_conv(self.x_rand) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 64)) + + def test_non_random_input(self): + out = self.depthwise_conv(self.x_ones) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 64)) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 64)) + + def test_zero_input(self): + out = self.depthwise_conv(self.x_zero) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 64)) + + def test_weight_change(self): + self.depthwise_conv.conv.weight.data.fill_(0.5) + out_first = self.depthwise_conv(self.x_rand) + + self.depthwise_conv.conv.weight.data.fill_(1.0) + out_second = self.depthwise_conv(self.x_rand) + + # Ensuring weight changes have an effect + self.assertTrue(torch.any(out_first != out_second)) + + +class TestPointwiseConv1d(unittest.TestCase): + def setUp(self): + # initialize parameters once and reuse them in multiple test cases + self.in_channels, self.out_channels, self.stride, self.padding, self.bias = ( + 2, + 4, + 1, + 1, + True, + ) + + self.pointwise_conv = PointwiseConv1d( + self.in_channels, self.out_channels, self.stride, self.padding, self.bias, + ) + + # Generate an input data + self.x_rand = torch.randn( + 32, + self.in_channels, + 64, + ) + self.x_zero = torch.zeros( + 32, + self.in_channels, + 64, + ) + self.x_ones = torch.ones( + 32, + self.in_channels, + 64, + ) + + def test_forward(self): + out = self.pointwise_conv(self.x_rand) + + self.assertIsInstance(out, torch.Tensor) + # Padding of 1 means one column of zeroes got added both at the beginning and at the end, + # that's why you have 64 (original) + 2 (padding) = 66 + self.assertEqual(out.shape, (32, self.out_channels, 66)) + + def test_non_random_input(self): + out = self.pointwise_conv(self.x_ones) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 66)) + + def test_zero_input(self): + out = self.pointwise_conv(self.x_zero) + + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (32, self.out_channels, 66)) + + def test_weight_change(self): + self.pointwise_conv.conv.weight.data.fill_(0.5) + out_first = self.pointwise_conv(self.x_rand) + + self.pointwise_conv.conv.weight.data.fill_(1.0) + out_second = self.pointwise_conv(self.x_rand) + + self.assertTrue(torch.any(out_first != out_second)) + + def test_kernel_size(self): + # Checking if the module can handle non-default kernel sizes. + kernel_size = 2 + pointwise_conv = PointwiseConv1d( + self.in_channels, + self.out_channels, + self.stride, + self.padding, + self.bias, + kernel_size, + ) + out = pointwise_conv(self.x_rand) + + # ((input_size - kernel_size + 2*padding)/stride ) + 1 + # ((64 - 2 + 2*1) / 1 ) + 1 = 65 + self.assertEqual(out.shape, (32, self.out_channels, 65)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_conv1d_glu.py b/models/tts/delightful_tts/conv_blocks/tests/test_conv1d_glu.py new file mode 100644 index 0000000000000000000000000000000000000000..4f3e1cdd29802c95a82916f9cb1d045a357fb60e --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_conv1d_glu.py @@ -0,0 +1,97 @@ +import unittest + +import torch +from torch import nn + +from models.tts.delightful_tts.conv_blocks.bsconv import BSConv1d +from models.tts.delightful_tts.conv_blocks.conv1d import ( + DepthWiseConv1d, + PointwiseConv1d, +) +from models.tts.delightful_tts.conv_blocks.conv1d_glu import Conv1dGLU + + +class TestConv1dGLU(unittest.TestCase): + def test_initialization(self): + """Test to check if the Conv1dGLU instance is properly created.""" + d_model, kernel_size, padding, embedding_dim = 16, 3, 1, 32 + conv_glu = Conv1dGLU( + d_model, + kernel_size, + padding, + embedding_dim, + ) + + # Checking attribute types + self.assertIsInstance( + conv_glu.bsconv1d, + BSConv1d, + msg="Expected bsconv1d attribute to be instance of BSConv1d Layer", + ) + self.assertIsInstance( + conv_glu.bsconv1d.pointwise, + PointwiseConv1d, + msg="Expected bsconv1d.pointwise to be instance of PointwiseConv1d", + ) + self.assertIsInstance( + conv_glu.bsconv1d.depthwise, + DepthWiseConv1d, + msg="Expected bsconv1d.depthwise to be instance of DepthWiseConv1d", + ) + self.assertIsInstance( + conv_glu.embedding_proj, + nn.Linear, + msg="Expected embedding_proj attribute to be instance of nn.Linear Layer", + ) + + # Checking the Conv1d, PointwiseConv1d, and DepthWiseConv1d configurations + self.assertEqual(conv_glu.bsconv1d.pointwise.conv.in_channels, d_model) + self.assertEqual(conv_glu.bsconv1d.pointwise.conv.out_channels, 2 * d_model) + self.assertEqual(conv_glu.bsconv1d.depthwise.conv.in_channels, 2 * d_model) + self.assertEqual(conv_glu.bsconv1d.depthwise.conv.out_channels, 2 * d_model) + self.assertEqual(conv_glu.bsconv1d.depthwise.conv.kernel_size[0], kernel_size) + self.assertEqual(conv_glu.bsconv1d.depthwise.conv.padding[0], padding) + + def test_output_shape(self): + """Test to ensure that the output tensor shape from the forward function is as expected.""" + batch_size, sequence_length, d_model, kernel_size, padding, embedding_dim = ( + 64, + 50, + 8, + 3, + 1, + 16, + ) + conv_glu = Conv1dGLU( + d_model, + kernel_size, + padding, + embedding_dim, + ) + + # Create input tensor 'x' of shape (batch_size, sequence_length, embed_dim) + x = torch.randn( + batch_size, + sequence_length, + d_model, + ) + + # Create embeddings tensor of shape (batch_size, d_model, embedding_dim) + embeddings = torch.randn( + batch_size, + sequence_length, + embedding_dim, + ) + + out = conv_glu(x, embeddings) + + # Assuming embeddings transformations don't change the width of the input + self.assertEqual( + out.shape, + (batch_size, sequence_length, d_model), + f"Expected output shape: {batch_size, sequence_length, d_model}, but got: {out.shape}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_conv_transposed.py b/models/tts/delightful_tts/conv_blocks/tests/test_conv_transposed.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc87a9e0457c7812585cdb3f72831e02b3204e4 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_conv_transposed.py @@ -0,0 +1,82 @@ +import unittest + +import torch + +from models.tts.delightful_tts.conv_blocks.bsconv import BSConv1d +from models.tts.delightful_tts.conv_blocks.conv_transposed import ConvTransposed + + +class TestConvTransposed(unittest.TestCase): + def test_initialization(self): + """Test to check if the ConvTransposed instance is properly created.""" + C_in, C_out, kernel_size, padding = 4, 6, 3, 1 + + # Initialize ConvTransposed with input channels, output channels, kernel size and padding. + conv_transposed = ConvTransposed( + C_in, + C_out, + kernel_size, + padding, + ) + + # Check type and value of 'conv' attribute which is an instance of BSConv1d + self.assertIsInstance( + conv_transposed.conv, + BSConv1d, + msg="Expected conv attribute to be instance of BSConv1d Layer", + ) + + # 'in_channels' and 'out_channels' attributes of conv.pointwise layer + self.assertEqual( + conv_transposed.conv.pointwise.conv.in_channels, + C_in, + msg=f"Expected conv.pointwise.conv.in_channels to be {C_in}, got {conv_transposed.conv.pointwise.conv.in_channels}", + ) + + self.assertEqual( + conv_transposed.conv.pointwise.conv.out_channels, + C_out, + msg=f"Expected conv.pointwise.conv.out_channels to be {C_out}, got {conv_transposed.conv.pointwise.conv.out_channels}", + ) + + # Configuration attributes of conv.depthwise layer + self.assertEqual( + conv_transposed.conv.depthwise.conv.kernel_size[0], + kernel_size, + msg=f"Expected conv.depthwise.conv.kernel_size[0] to be {kernel_size}, got {conv_transposed.conv.depthwise.conv.kernel_size[0]}", + ) + + self.assertEqual( + conv_transposed.conv.depthwise.conv.padding[0], + padding, + msg=f"Expected conv.depthwise.conv.padding[0] to be {padding}, got {conv_transposed.conv.depthwise.conv.padding[0]}", + ) + + def test_shape(self): + """Test to check if the ConvTransposed instance is properly created and if the output shape is as expected.""" + N, C_in, C_out, kernel_size, padding, length = 5, 4, 6, 3, 1, 10 + + # Initialize ConvTransposed with input channels, output channels, kernel size and padding. + conv_transposed = ConvTransposed( + C_in, + C_out, + kernel_size, + padding, + ) + + # Generate a random tensor of shape (batch_size, channel, length). + x = torch.randn( + N, + length, + C_in, + ) + + out = conv_transposed(x) + + # The output shape should be the same as the input shape, as the ConvTransposed should not + # alter the input's dimensions. + self.assertEqual(out.shape, (N, length, C_out)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/conv_blocks/tests/test_coord_conv1d.py b/models/tts/delightful_tts/conv_blocks/tests/test_coord_conv1d.py new file mode 100644 index 0000000000000000000000000000000000000000..3451b52ae595ecfabc109e06f345b2b2bea59140 --- /dev/null +++ b/models/tts/delightful_tts/conv_blocks/tests/test_coord_conv1d.py @@ -0,0 +1,65 @@ +import unittest + +import torch + +from models.tts.delightful_tts.conv_blocks.coord_conv1d import CoordConv1d + + +class TestCoordConv1d(unittest.TestCase): + def setUp(self): + self.x_rand = torch.randn(1, 2, 10) + + def test_simple_case(self): + """Tests a simple case with input of size (1, 2, 10)""" + model = CoordConv1d( + in_channels=2, + out_channels=8, + kernel_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + with_r=False, + ) + output = model(self.x_rand) + + self.assertEqual(list(output.shape), [1, 8, 8]) + + def test_with_r(self): + """Tests if 'with_r' adds an extra radial channel""" + model = CoordConv1d( + in_channels=2, + out_channels=8, + kernel_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + with_r=True, + ) + output = model(self.x_rand) + + self.assertEqual(list(output.shape), [1, 8, 8]) + + def test_with_padding(self): + """Tests if padding is functioning correctly""" + model = CoordConv1d( + in_channels=2, + out_channels=8, + kernel_size=3, + stride=1, + padding=1, + dilation=1, + groups=1, + bias=True, + with_r=False, + ) + output = model(self.x_rand) + self.assertEqual(list(output.shape), [1, 8, 10]) + + +# To run the tests +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/delightful_tts.py b/models/tts/delightful_tts/delightful_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..ff9bff740ab3b559c0390971b8ecec8e3658e11a --- /dev/null +++ b/models/tts/delightful_tts/delightful_tts.py @@ -0,0 +1,349 @@ +from typing import List + +from lightning.pytorch.core import LightningModule +import torch +from torch import Tensor +from torch.optim import AdamW +from torch.optim.lr_scheduler import ExponentialLR +from torch.utils.data import DataLoader + +from models.config import ( + AcousticFinetuningConfig, + AcousticModelConfigType, + AcousticMultilingualModelConfig, + AcousticPretrainingConfig, + AcousticTrainingConfig, + PreprocessingConfig, + get_lang_map, + lang2id, +) +from models.helpers.tools import get_mask_from_lengths +from training.datasets.hifi_libri_dataset import ( + speakers_hifi_ids, + speakers_libri_ids, + train_dataloader, +) +from training.loss import FastSpeech2LossGen +from training.preprocess.normalize_text import NormalizeText + +# Updated version of the tokenizer +from training.preprocess.tokenizer_ipa_espeak import TokenizerIpaEspeak as TokenizerIPA + +from .acoustic_model import AcousticModel + +MEL_SPEC_EVERY_N_STEPS = 1000 +AUDIO_EVERY_N_STEPS = 100 + + +class DelightfulTTS(LightningModule): + r"""Trainer for the acoustic model. + + Args: + preprocess_config PreprocessingConfig: The preprocessing configuration. + model_config AcousticModelConfigType: The model configuration. + fine_tuning (bool, optional): Whether to use fine-tuning mode or not. Defaults to False. + bin_warmup (bool, optional): Whether to use binarization warmup for the loss or not. Defaults to True. + lang (str): Language of the dataset. + n_speakers (int): Number of speakers in the dataset.generation during training. + batch_size (int): The batch size. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType = AcousticMultilingualModelConfig(), + fine_tuning: bool = False, + bin_warmup: bool = True, + lang: str = "en", + n_speakers: int = 5392, + batch_size: int = 19, + ): + super().__init__() + + self.lang = lang + self.lang_id = lang2id[self.lang] + + self.fine_tuning = fine_tuning + self.batch_size = batch_size + + lang_map = get_lang_map(lang) + normilize_text_lang = lang_map.nemo + + self.tokenizer = TokenizerIPA(lang) + self.normilize_text = NormalizeText(normilize_text_lang) + + self.train_config_acoustic: AcousticTrainingConfig + + if self.fine_tuning: + self.train_config_acoustic = AcousticFinetuningConfig() + else: + self.train_config_acoustic = AcousticPretrainingConfig() + + self.preprocess_config = preprocess_config + + # TODO: fix the arguments! + self.acoustic_model = AcousticModel( + preprocess_config=self.preprocess_config, + model_config=model_config, + # NOTE: this parameter may be hyperparameter that you can define based on the demands + n_speakers=n_speakers, + ) + + # NOTE: in case of training from 0 bin_warmup should be True! + self.loss_acoustic = FastSpeech2LossGen( + bin_warmup=bin_warmup, + ) + + def forward( + self, + text: str, + speaker_idx: Tensor, + ) -> Tensor: + r"""Performs a forward pass through the AcousticModel. + This code must be run only with the loaded weights from the checkpoint! + + Args: + text (str): The input text. + speaker_idx (Tensor): The index of the speaker + + Returns: + Tensor: The generated waveform with hifi-gan. + """ + normalized_text = self.normilize_text(text) + _, phones = self.tokenizer(normalized_text) + + # Convert to tensor + x = torch.tensor( + phones, + dtype=torch.int, + device=speaker_idx.device, + ).unsqueeze(0) + + speakers = speaker_idx.repeat(x.shape[1]).unsqueeze(0) + + langs = ( + torch.tensor( + [self.lang_id], + dtype=torch.int, + device=speaker_idx.device, + ) + .repeat(x.shape[1]) + .unsqueeze(0) + ) + + mel_pred = self.acoustic_model.forward( + x=x, + speakers=speakers, + langs=langs, + ) + + return mel_pred + + def training_step(self, batch: List, _: int): + r"""Performs a training step for the model. + + Args: + batch (List): The batch of data for training. The batch should contain: + - ids: List of indexes. + - raw_texts: Raw text inputs. + - speakers: Speaker identities. + - texts: Text inputs. + - src_lens: Lengths of the source sequences. + - mels: Mel spectrogram targets. + - pitches: Pitch targets. + - pitches_stat: Statistics of the pitches. + - mel_lens: Lengths of the mel spectrograms. + - langs: Language identities. + - attn_priors: Prior attention weights. + - wavs: Waveform targets. + - energies: Energy targets. + batch_idx (int): Index of the batch. + + Returns: + - 'loss': The total loss for the training step. + """ + ( + _, + _, + speakers, + texts, + src_lens, + mels, + pitches, + _, + mel_lens, + langs, + attn_priors, + _, + energies, + ) = batch + + outputs = self.acoustic_model.forward_train( + x=texts, + speakers=speakers, + src_lens=src_lens, + mels=mels, + mel_lens=mel_lens, + pitches=pitches, + langs=langs, + attn_priors=attn_priors, + energies=energies, + ) + + y_pred = outputs["y_pred"] + log_duration_prediction = outputs["log_duration_prediction"] + p_prosody_ref = outputs["p_prosody_ref"] + p_prosody_pred = outputs["p_prosody_pred"] + pitch_prediction = outputs["pitch_prediction"] + energy_pred = outputs["energy_pred"] + energy_target = outputs["energy_target"] + + src_mask = get_mask_from_lengths(src_lens) + mel_mask = get_mask_from_lengths(mel_lens) + + ( + total_loss, + mel_loss, + ssim_loss, + duration_loss, + u_prosody_loss, + p_prosody_loss, + pitch_loss, + ctc_loss, + bin_loss, + energy_loss, + ) = self.loss_acoustic.forward( + src_masks=src_mask, + mel_masks=mel_mask, + mel_targets=mels, + mel_predictions=y_pred, + log_duration_predictions=log_duration_prediction, + u_prosody_ref=outputs["u_prosody_ref"], + u_prosody_pred=outputs["u_prosody_pred"], + p_prosody_ref=p_prosody_ref, + p_prosody_pred=p_prosody_pred, + pitch_predictions=pitch_prediction, + p_targets=outputs["pitch_target"], + durations=outputs["attn_hard_dur"], + attn_logprob=outputs["attn_logprob"], + attn_soft=outputs["attn_soft"], + attn_hard=outputs["attn_hard"], + src_lens=src_lens, + mel_lens=mel_lens, + energy_pred=energy_pred, + energy_target=energy_target, + step=self.trainer.global_step, + ) + + self.log( + "train_total_loss", + total_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log("train_mel_loss", mel_loss, sync_dist=True, batch_size=self.batch_size) + self.log( + "train_ssim_loss", + ssim_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "train_duration_loss", + duration_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "train_u_prosody_loss", + u_prosody_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "train_p_prosody_loss", + p_prosody_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "train_pitch_loss", + pitch_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log("train_ctc_loss", ctc_loss, sync_dist=True, batch_size=self.batch_size) + self.log("train_bin_loss", bin_loss, sync_dist=True, batch_size=self.batch_size) + self.log( + "train_energy_loss", + energy_loss, + sync_dist=True, + batch_size=self.batch_size, + ) + + return total_loss + + def configure_optimizers(self): + r"""Configures the optimizer used for training. + + Returns + tuple: A tuple containing three dictionaries. Each dictionary contains the optimizer and learning rate scheduler for one of the models. + """ + lr_decay = self.train_config_acoustic.optimizer_config.lr_decay + default_lr = self.train_config_acoustic.optimizer_config.learning_rate + + init_lr = ( + default_lr + if self.trainer.global_step == 0 + else default_lr * (lr_decay**self.trainer.global_step) + ) + + optimizer_acoustic = AdamW( + self.acoustic_model.parameters(), + lr=init_lr, + betas=self.train_config_acoustic.optimizer_config.betas, + eps=self.train_config_acoustic.optimizer_config.eps, + weight_decay=self.train_config_acoustic.optimizer_config.weight_decay, + ) + + scheduler_acoustic = ExponentialLR(optimizer_acoustic, gamma=lr_decay) + + return { + "optimizer": optimizer_acoustic, + "lr_scheduler": scheduler_acoustic, + } + + def train_dataloader( + self, + root: str = "datasets_cache", + cache: bool = True, + cache_dir: str = "/dev/shm", + include_libri: bool = False, + libri_speakers: List[str] = speakers_libri_ids, + hifi_speakers: List[str] = speakers_hifi_ids, + ) -> DataLoader: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. Defaults to "/dev/shm". + include_libri (bool): Whether to include the LibriTTS dataset or not. + libri_speakers (List[str]): The list of LibriTTS speakers to include. + hifi_speakers (List[str]): The list of HiFi-GAN speakers to include. + + Returns: + Tupple[DataLoader, DataLoader]: The training and validation dataloaders. + """ + return train_dataloader( + batch_size=self.batch_size, + num_workers=self.preprocess_config.workers, + sampling_rate=self.preprocess_config.sampling_rate, + root=root, + cache=cache, + cache_dir=cache_dir, + lang=self.lang, + include_libri=include_libri, + libri_speakers=libri_speakers, + hifi_speakers=hifi_speakers, + ) diff --git a/models/tts/delightful_tts/reference_encoder/STL.py b/models/tts/delightful_tts/reference_encoder/STL.py new file mode 100644 index 0000000000000000000000000000000000000000..f8f47c800e930699f0e86cf12484ece3cbe00d3f --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/STL.py @@ -0,0 +1,73 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import AcousticModelConfigType +from models.tts.delightful_tts.attention import StyleEmbedAttention + + +class STL(Module): + r"""Style Token Layer (STL). + This layer helps to encapsulate different speaking styles in token embeddings. + + Args: + model_config (AcousticModelConfigType): An object containing the model's configuration parameters. + + Attributes: + embed (nn.Parameter): The style token embedding tensor. + attention (StyleEmbedAttention): The attention module used to compute a weighted sum of embeddings. + """ + + def __init__( + self, + model_config: AcousticModelConfigType, + ): + super().__init__() + + # Number of attention heads + num_heads = 1 + # Dimension of encoder hidden states + n_hidden = model_config.encoder.n_hidden + # Number of style tokens + self.token_num = model_config.reference_encoder.token_num + + # Define a learnable tensor for style tokens embedding + self.embed = nn.Parameter( + torch.FloatTensor(self.token_num, n_hidden // num_heads), + ) + + # Dimension of query in attention + d_q = n_hidden // 2 + # Dimension of keys in attention + d_k = n_hidden // num_heads + + # Style Embedding Attention module + self.attention = StyleEmbedAttention( + query_dim=d_q, + key_dim=d_k, + num_units=n_hidden, + num_heads=num_heads, + ) + + # Initialize the embedding with normal distribution + torch.nn.init.normal_(self.embed, mean=0, std=0.5) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the Style Token Layer + Args: + x (torch.Tensor): The input tensor. + + Returns + torch.Tensor: The emotion embedded tensor after applying attention mechanism. + """ + N = x.size(0) + + # Reshape input tensor to [N, 1, n_hidden // 2] + query = x.unsqueeze(1) + + keys_soft = ( + torch.tanh(self.embed).unsqueeze(0).expand(N, -1, -1) + ) # [N, token_num, n_hidden // num_heads] + + # Apply attention mechanism to get weighted sum of style token embeddings + return self.attention(query, keys_soft) diff --git a/models/tts/delightful_tts/reference_encoder/__init__.py b/models/tts/delightful_tts/reference_encoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d00ba849bb6d63441751d0e018d8890fa77ff65 --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/__init__.py @@ -0,0 +1,4 @@ +from .phoneme_level_prosody_encoder import PhonemeLevelProsodyEncoder +from .reference_encoder import ReferenceEncoder +from .STL import STL +from .utterance_level_prosody_encoder import UtteranceLevelProsodyEncoder diff --git a/models/tts/delightful_tts/reference_encoder/phoneme_level_prosody_encoder.py b/models/tts/delightful_tts/reference_encoder/phoneme_level_prosody_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..00edecb4c6eecd0b9b05637ab20c9cba421ed631 --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/phoneme_level_prosody_encoder.py @@ -0,0 +1,91 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import ( + AcousticModelConfigType, + PreprocessingConfig, +) +from models.tts.delightful_tts.attention import ConformerMultiHeadedSelfAttention + +from .reference_encoder import ReferenceEncoder + + +class PhonemeLevelProsodyEncoder(Module): + r"""Phoneme Level Prosody Encoder Module + + This Class is used to encode the phoneme level prosody in the speech synthesis pipeline. + + Args: + preprocess_config (PreprocessingConfig): Configuration for preprocessing. + model_config (AcousticModelConfigType): Acoustic model configuration. + + Returns: + torch.Tensor: The encoded tensor after applying masked fill. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType, + ): + super().__init__() + + # Obtain the bottleneck size and reference encoder gru size from the model config. + bottleneck_size = model_config.reference_encoder.bottleneck_size_p + ref_enc_gru_size = model_config.reference_encoder.ref_enc_gru_size + + # Initialize ReferenceEncoder, Linear layer and ConformerMultiHeadedSelfAttention for attention mechanism. + self.encoder = ReferenceEncoder(preprocess_config, model_config) + self.encoder_prj = nn.Linear(ref_enc_gru_size, model_config.encoder.n_hidden) + self.attention = ConformerMultiHeadedSelfAttention( + d_model=model_config.encoder.n_hidden, + num_heads=model_config.encoder.n_heads, + dropout_p=model_config.encoder.p_dropout, + ) + + # Bottleneck layer to transform the output of the attention mechanism. + self.encoder_bottleneck = nn.Linear( + model_config.encoder.n_hidden, bottleneck_size, + ) + + def forward( + self, + x: torch.Tensor, + src_mask: torch.Tensor, + mels: torch.Tensor, + mel_lens: torch.Tensor, + encoding: torch.Tensor, + ) -> torch.Tensor: + r"""The forward pass of the PhonemeLevelProsodyEncoder. Input tensors are passed through the reference encoder, + attention mechanism, and a bottleneck. + + Args: + x (torch.Tensor): Input tensor of shape [N, seq_len, encoder_embedding_dim]. + src_mask (torch.Tensor): The mask tensor which contains `True` at positions where the input x has been masked. + mels (torch.Tensor): The mel-spectrogram with shape [N, Ty/r, n_mels*r], where r=1. + mel_lens (torch.Tensor): The lengths of each sequence in mels. + encoding (torch.Tensor): The relative positional encoding tensor. + + Returns: + torch.Tensor: Output tensor of shape [N, seq_len, bottleneck_size]. + """ + # Use the reference encoder to embed prosody representation + embedded_prosody, _, mel_masks = self.encoder(mels, mel_lens) + + # Pass the prosody representation through a bottleneck (dimension reduction) + embedded_prosody = self.encoder_prj(embedded_prosody) + + # Flatten and apply attention mask + attn_mask = mel_masks.view((mel_masks.shape[0], 1, 1, -1)) + x, _ = self.attention( + query=x, + key=embedded_prosody, + value=embedded_prosody, + mask=attn_mask, + encoding=encoding, + ) + + # Apply the bottleneck to the output and mask out irrelevant parts + x = self.encoder_bottleneck(x) + return x.masked_fill(src_mask.unsqueeze(-1), 0.0) diff --git a/models/tts/delightful_tts/reference_encoder/reference_encoder.py b/models/tts/delightful_tts/reference_encoder/reference_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..14d3b1be1e5f540c59491fb350bdbcd1a0251981 --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/reference_encoder.py @@ -0,0 +1,170 @@ +from typing import Tuple + +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F + +from models.config import ( + AcousticModelConfigType, + PreprocessingConfig, +) +from models.helpers import tools +from models.tts.delightful_tts.constants import LEAKY_RELU_SLOPE +from models.tts.delightful_tts.conv_blocks import CoordConv1d + + +class ReferenceEncoder(Module): + r"""A class to define the reference encoder. + Similar to Tacotron model, the reference encoder is used to extract the high-level features from the reference + + It consists of a number of convolutional blocks (`CoordConv1d` for the first one and `nn.Conv1d` for the rest), + then followed by instance normalization and GRU layers. + The `CoordConv1d` at the first layer to better preserve positional information, paper: + [Robust and fine-grained prosody control of end-to-end speech synthesis](https://arxiv.org/pdf/1811.02122.pdf) + + Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence. + + Args: + preprocess_config (PreprocessingConfig): Configuration object with preprocessing parameters. + model_config (AcousticModelConfigType): Configuration object with acoustic model parameters. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing three tensors. _First_: The sequence tensor + produced by the last GRU layer after padding has been removed. _Second_: The GRU's final hidden state tensor. + _Third_: The mask tensor, which has the same shape as x, and contains `True` at positions where the input x + has been masked. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType, + ): + super().__init__() + + n_mel_channels = preprocess_config.stft.n_mel_channels + ref_enc_filters = model_config.reference_encoder.ref_enc_filters + ref_enc_size = model_config.reference_encoder.ref_enc_size + ref_enc_strides = model_config.reference_encoder.ref_enc_strides + ref_enc_gru_size = model_config.reference_encoder.ref_enc_gru_size + + self.n_mel_channels = n_mel_channels + K = len(ref_enc_filters) + filters = [self.n_mel_channels, *ref_enc_filters] + strides = [1, *ref_enc_strides] + + # Use CoordConv1d at the first layer to better preserve positional information: https://arxiv.org/pdf/1811.02122.pdf + convs = [ + CoordConv1d( + in_channels=filters[0], + out_channels=filters[0 + 1], + kernel_size=ref_enc_size, + stride=strides[0], + padding=ref_enc_size // 2, + with_r=True, + ), + *[ + nn.Conv1d( + in_channels=filters[i], + out_channels=filters[i + 1], + kernel_size=ref_enc_size, + stride=strides[i], + padding=ref_enc_size // 2, + ) + for i in range(1, K) + ], + ] + # Define convolution layers (ModuleList) + self.convs = nn.ModuleList(convs) + + self.norms = nn.ModuleList( + [ + nn.InstanceNorm1d(num_features=ref_enc_filters[i], affine=True) + for i in range(K) + ], + ) + + # Define GRU layer + self.gru = nn.GRU( + input_size=ref_enc_filters[-1], + hidden_size=ref_enc_gru_size, + batch_first=True, + ) + + def forward( + self, + x: torch.Tensor, + mel_lens: torch.Tensor, + leaky_relu_slope: float = LEAKY_RELU_SLOPE, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Forward pass of the ReferenceEncoder. + + Args: + x (torch.Tensor): A 3-dimensional tensor containing the input sequences, its size is [N, n_mels, timesteps]. + mel_lens (torch.Tensor): A 1-dimensional tensor containing the lengths of each sequence in x. Its length is N. + leaky_relu_slope (float): The slope of the leaky relu function. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing three tensors. _First_: The sequence tensor + produced by the last GRU layer after padding has been removed. _Second_: The GRU's final hidden state tensor. + _Third_: The mask tensor, which has the same shape as x, and contains `True` at positions where the input x + has been masked. + """ + mel_masks = tools.get_mask_from_lengths(mel_lens).unsqueeze(1) + mel_masks = mel_masks.to(x.device) + + x = x.masked_fill(mel_masks, 0) + for conv, norm in zip(self.convs, self.norms): + x = x.float() + x = conv(x) + x = F.leaky_relu(x, leaky_relu_slope) # [N, 128, Ty//2^K, n_mels//2^K] + x = norm(x) + + for _ in range(2): + mel_lens = tools.stride_lens_downsampling(mel_lens) + + mel_masks = tools.get_mask_from_lengths(mel_lens) + + x = x.masked_fill(mel_masks.unsqueeze(1), 0) + x = x.permute((0, 2, 1)) + + packed_sequence = torch.nn.utils.rnn.pack_padded_sequence( + x, + lengths=mel_lens.cpu().int(), + batch_first=True, + enforce_sorted=False, + ) + + self.gru.flatten_parameters() + # memory --- [N, Ty, E//2], out --- [1, N, E//2] + out, memory = self.gru(packed_sequence) + out, _ = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first=True) + + return out, memory, mel_masks + + def calculate_channels( + self, + L: int, + kernel_size: int, + stride: int, + pad: int, + n_convs: int, + ) -> int: + r"""Calculate the number of channels after applying convolutions. + + Args: + L (int): The original size. + kernel_size (int): The kernel size used in the convolutions. + stride (int): The stride used in the convolutions. + pad (int): The padding used in the convolutions. + n_convs (int): The number of convolutions. + + Returns: + int: The size after the convolutions. + """ + # Loop through each convolution + for _ in range(n_convs): + # Calculate the size after each convolution + L = (L - kernel_size + 2 * pad) // stride + 1 + return L diff --git a/models/tts/delightful_tts/reference_encoder/tests/__init__.py b/models/tts/delightful_tts/reference_encoder/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/tts/delightful_tts/reference_encoder/tests/test_STL.py b/models/tts/delightful_tts/reference_encoder/tests/test_STL.py new file mode 100644 index 0000000000000000000000000000000000000000..49a21d2c6475702073aeb3b483742f08ae9ae685 --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/tests/test_STL.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import Mock + +import torch + +from models.tts.delightful_tts.reference_encoder.STL import STL + + +class TestSTL(unittest.TestCase): + def setUp(self): + self.model_config = Mock() + self.model_config.encoder.n_hidden = 512 + self.model_config.reference_encoder.token_num = 32 + + self.stl = STL( + self.model_config, + ) + self.batch_size = 10 + self.n_hidden = self.model_config.encoder.n_hidden + + self.x = torch.rand( + self.batch_size, + self.n_hidden // 2, + ) + + def test_forward(self): + output = self.stl(self.x) + + self.assertTrue(torch.is_tensor(output)) + + # Validate the output size + expected_shape = (self.batch_size, 1, self.stl.attention.num_units) + self.assertEqual(output.shape, expected_shape) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/reference_encoder/tests/test_phoneme_level_prosody_encoder.py b/models/tts/delightful_tts/reference_encoder/tests/test_phoneme_level_prosody_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..af4a67449c876d5b0757b17875a529796b836362 --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/tests/test_phoneme_level_prosody_encoder.py @@ -0,0 +1,148 @@ +import unittest + +import torch +from torch import nn + +from models.config import ( + AcousticENModelConfig, + AcousticPretrainingConfig, +) +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.helpers.initializer import ( + init_acoustic_model, + init_conformer, + init_forward_trains_params, +) +from models.tts.delightful_tts.attention.conformer_multi_headed_self_attention import ( + ConformerMultiHeadedSelfAttention, +) +from models.tts.delightful_tts.reference_encoder import ( + PhonemeLevelProsodyEncoder, + UtteranceLevelProsodyEncoder, +) +from models.tts.delightful_tts.reference_encoder.reference_encoder import ( + ReferenceEncoder, +) + + +# It checks for most of the acoustic model code +# Here you can understand the input and output shapes of the PhonemeLevelProsodyEncoder +# Integration test +class TestPhonemeLevelProsodyEncoder(unittest.TestCase): + def setUp(self): + self.acoustic_pretraining_config = AcousticPretrainingConfig() + self.model_config = AcousticENModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + # Based on speaker.json mock + n_speakers = 10 + + # # Add Conformer as encoder + self.encoder, _ = init_conformer(self.model_config) + + # Add AcousticModel instance + self.acoustic_model, _ = init_acoustic_model( + self.preprocess_config, + self.model_config, + n_speakers, + ) + + # Generate mock data for the forward pass + self.forward_train_params = init_forward_trains_params( + self.model_config, + self.acoustic_pretraining_config, + self.preprocess_config, + n_speakers, + ) + + preprocess_config = self.preprocess_config + model_config = self.model_config + + self.utterance_prosody_encoder = UtteranceLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.phoneme_prosody_encoder = PhonemeLevelProsodyEncoder( + preprocess_config, + model_config, + ) + + self.u_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_u, + elementwise_affine=False, + ) + + self.p_norm = nn.LayerNorm( + model_config.reference_encoder.bottleneck_size_p, + elementwise_affine=False, + ) + + self.model = PhonemeLevelProsodyEncoder( + self.preprocess_config, + self.model_config, + ) + + def test_model_attributes(self): + # Test model type + self.assertIsInstance(self.model, nn.Module) + + # Test individual components of the model + self.assertIsInstance(self.model.encoder, ReferenceEncoder) + self.assertIsInstance(self.model.encoder_prj, nn.Linear) + self.assertIsInstance(self.model.attention, ConformerMultiHeadedSelfAttention) + self.assertIsInstance(self.model.encoder_bottleneck, nn.Linear) + + def test_forward(self): + x = torch.randn(1, 11, self.model_config.encoder.n_hidden) + mels = torch.randn(1, self.preprocess_config.stft.n_mel_channels, 58) + mel_lens = torch.tensor([58]) + src_mask = torch.zeros(11).bool() + encoding = torch.randn(1, 58, self.model_config.encoder.n_hidden) + + u_prosody_ref = self.u_norm( + self.utterance_prosody_encoder( + mels=mels, + mel_lens=mel_lens, + ), + ) + + # Assert the shape of u_prosody_ref + self.assertEqual( + u_prosody_ref.shape, + torch.Size( + [ + self.model_config.lang_embed_dim, + self.model_config.lang_embed_dim, + self.model_config.reference_encoder.bottleneck_size_u, + ], + ), + ) + + p_prosody_ref = self.p_norm( + self.phoneme_prosody_encoder( + x=x, + src_mask=src_mask, + mels=mels, + mel_lens=mel_lens, + encoding=encoding, + ), + ) + + # Assert the shape of p_prosody_ref + self.assertEqual( + p_prosody_ref.shape, + torch.Size( + [ + self.model_config.lang_embed_dim, + 11, + self.model_config.reference_encoder.bottleneck_size_p, + ], + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/reference_encoder/tests/test_reference_encoder.py b/models/tts/delightful_tts/reference_encoder/tests/test_reference_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..eb15698444be2879bb6f3112f15240b89f463a6f --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/tests/test_reference_encoder.py @@ -0,0 +1,68 @@ +import unittest + +import torch + +from models.config import AcousticENModelConfig +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.tts.delightful_tts.reference_encoder import ReferenceEncoder + + +class TestReferenceEncoder(unittest.TestCase): + def setUp(self): + self.preprocess_config = PreprocessingConfig("english_only") + self.model_config = AcousticENModelConfig() + + self.model = ReferenceEncoder( + self.preprocess_config, + self.model_config, + ) + + def test_forward_shape(self): + # Define test case + x = torch.randn( + 16, + self.model.n_mel_channels, + 128, + ) # assuming the input sequence length is 128 + mel_lens = ( + torch.ones(16, dtype=torch.long) * 128 + ) # assuming all sequences are of equal length + + # Call the forward method + out, memory, mel_masks = self.model(x, mel_lens) + + # Verify the outputs + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.size(0), 16) + self.assertEqual(out.size(2), self.model.gru.hidden_size) + + self.assertIsInstance(memory, torch.Tensor) + self.assertEqual(memory.size(0), 1) + self.assertEqual(memory.size(2), self.model.gru.hidden_size) + + self.assertIsInstance(mel_masks, torch.Tensor) + self.assertEqual(list(mel_masks.size()), [16, 32]) + + def test_different_batch_sizes(self): + for batch_size in [1, 5, 10, 50]: + x = torch.randn( + batch_size, + self.model.n_mel_channels, + 128, + ) + mel_lens = ( + torch.ones( + batch_size, + dtype=torch.long, + ) + * 128 + ) + + out, memory, mel_masks = self.model(x, mel_lens) + + self.assertEqual(out.size(0), batch_size) + self.assertEqual(out.size(2), self.model.gru.hidden_size) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/reference_encoder/tests/test_utterance_level_prosody_encoder.py b/models/tts/delightful_tts/reference_encoder/tests/test_utterance_level_prosody_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..3caeb9166d2f4c0528c5be4edb89984fc6ec3c8d --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/tests/test_utterance_level_prosody_encoder.py @@ -0,0 +1,41 @@ +import unittest + +import torch + +from models.config import AcousticENModelConfig +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.tts.delightful_tts.reference_encoder import UtteranceLevelProsodyEncoder + + +class TestUtteranceLevelProsodyEncoder(unittest.TestCase): + def setUp(self): + self.preprocess_config = PreprocessingConfig("english_only") + self.model_config = AcousticENModelConfig() + + # Instantiate the model to be tested with the mock configurations + self.model = UtteranceLevelProsodyEncoder( + self.preprocess_config, + self.model_config, + ) + + def test_forward_shape(self): + # Define the input tensor (mels) and corresponding lengths (mel_lens) + mels = torch.randn( + 16, + 100, + 128, + ) # assuming the input sequence length is 128 and n_mel_channels=100 + mel_lens = ( + torch.ones(16, dtype=torch.long) * 128 + ) # assuming all sequences are of equal length + + # Make a forward pass through the model + out = self.model(mels, mel_lens) + + # Assert the shape of the output tensor + self.assertIsInstance(out, torch.Tensor) + self.assertEqual(out.shape, (16, 1, self.model.encoder_bottleneck.out_features)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/tts/delightful_tts/reference_encoder/utterance_level_prosody_encoder.py b/models/tts/delightful_tts/reference_encoder/utterance_level_prosody_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..897c39ede458b4725e18d21a0965c812f965e62f --- /dev/null +++ b/models/tts/delightful_tts/reference_encoder/utterance_level_prosody_encoder.py @@ -0,0 +1,71 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import ( + AcousticModelConfigType, + PreprocessingConfig, +) + +from .reference_encoder import ReferenceEncoder +from .STL import STL + + +class UtteranceLevelProsodyEncoder(Module): + r"""A class to define the utterance level prosody encoder. + + The encoder uses a Reference encoder class to convert input sequences into high-level features, + followed by prosody embedding, self attention on the embeddings, and a feedforward transformation to generate the final output.Initializes the encoder with given specifications and creates necessary layers. + + Args: + preprocess_config (PreprocessingConfig): Configuration object with preprocessing parameters. + model_config (AcousticModelConfigType): Configuration object with acoustic model parameters. + + Returns: + torch.Tensor: A 3-dimensional tensor sized `[N, seq_len, E]`. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + model_config: AcousticModelConfigType, + ): + super().__init__() + + self.E = model_config.encoder.n_hidden + ref_enc_gru_size = model_config.reference_encoder.ref_enc_gru_size + ref_attention_dropout = model_config.reference_encoder.ref_attention_dropout + bottleneck_size = model_config.reference_encoder.bottleneck_size_u + + # Define important layers/modules for the encoder + self.encoder = ReferenceEncoder(preprocess_config, model_config) + self.encoder_prj = nn.Linear(ref_enc_gru_size, self.E // 2) + self.stl = STL(model_config) + self.encoder_bottleneck = nn.Linear(self.E, bottleneck_size) + self.dropout = nn.Dropout(ref_attention_dropout) + + def forward(self, mels: torch.Tensor, mel_lens: torch.Tensor) -> torch.Tensor: + r"""Defines the forward pass of the utterance level prosody encoder. + + Args: + mels (torch.Tensor): A 3-dimensional tensor containing input sequences. Size is `[N, Ty/r, n_mels*r]`. + mel_lens (torch.Tensor): A 1-dimensional tensor containing the lengths of each sequence in mels. Length is N. + + Returns: + torch.Tensor: A 3-dimensional tensor sized `[N, seq_len, E]`. + """ + # Use the reference encoder to get prosody embeddings + _, embedded_prosody, _ = self.encoder(mels, mel_lens) + + # Bottleneck + # Use the linear projection layer on the prosody embeddings + embedded_prosody = self.encoder_prj(embedded_prosody) + + # Apply the style token layer followed by the bottleneck layer + out = self.encoder_bottleneck(self.stl(embedded_prosody)) + + # Apply dropout for regularization + out = self.dropout(out) + + # Reshape the output tensor before returning + return out.view((-1, 1, out.shape[3])) diff --git a/models/tts/delightful_tts/tests/__init__.py b/models/tts/delightful_tts/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/tts/delightful_tts/tests/mock.py b/models/tts/delightful_tts/tests/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..8865617b0001eae26ef53dbc76980526495c5ab4 --- /dev/null +++ b/models/tts/delightful_tts/tests/mock.py @@ -0,0 +1,15 @@ +import torch + + +def get_dummy_input(device: torch.device): + text = torch.tensor([ + 2, 42, 14, 44, 22, 50, 21, 10, 42, 27, 24, 36, 19, 16, 42, 32, 20, 4, 42, 19, 37, 16, 19, 28, 32, 4, 45, 21, 21, 22, 50, 37, 14, 39, 50, 21, 30, 37, 44, 42, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], device=device) + src_len = torch.tensor([42], device=device) + speakers = torch.tensor([ + 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, 2436, + ], device=device) + langs = torch.tensor([ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + ], device=device) + return text, src_len, speakers, langs diff --git a/models/tts/delightful_tts/tests/test_delightful_tts.py b/models/tts/delightful_tts/tests/test_delightful_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..70600c6692411bc3c704f6e208823d4825ba5331 --- /dev/null +++ b/models/tts/delightful_tts/tests/test_delightful_tts.py @@ -0,0 +1,98 @@ +import os +import unittest + +from lightning.pytorch import Trainer +import torch + +from models.config import PreprocessingConfigHifiGAN as PreprocessingConfig +from models.config import PreprocessingConfigUnivNet +from models.tts.delightful_tts.delightful_tts import DelightfulTTS +from models.vocoder.univnet import UnivNet + +checkpoint = "checkpoints/logs_44100_tts_80_logs_new3_lightning_logs_version_7_checkpoints_epoch=2450-step=183470.ckpt" + +# NOTE: this is needed to avoid CUDA_LAUNCH_BLOCKING error +os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + + +class TestDelightfulTTS(unittest.TestCase): + def setUp(self): + # Create a dummy Trainer instance + self.trainer = Trainer() + self.preprocessing_config = PreprocessingConfig("multilingual") + + def test_optim_finetuning(self): + module = DelightfulTTS(preprocess_config=self.preprocessing_config) + + module.trainer = self.trainer + + optimizer_config = module.configure_optimizers() + + optimizer = optimizer_config["optimizer"] + lr_scheduler = optimizer_config["lr_scheduler"] + + # Test the optimizer + self.assertIsInstance(optimizer, torch.optim.AdamW) + self.assertIsInstance(lr_scheduler, torch.optim.lr_scheduler.ExponentialLR) + + def test_train_steps(self): + default_root_dir = "checkpoints/acoustic" + # checkpoint = "checkpoints/logs_new_training_libri-360-swa_multilingual_conf_epoch=146-step=33516.ckpt" + + trainer = Trainer( + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + fast_dev_run=1, + limit_train_batches=1, + max_epochs=1, + accelerator="gpu", + ) + + module = DelightfulTTS( + preprocess_config=self.preprocessing_config, + ) + + train_dataloader = module.train_dataloader(cache=False) + + # automatically restores model, epoch, step, LR schedulers, etc... + # trainer.fit(model, ckpt_path="some/path/to/my_checkpoint.ckpt") + + result = trainer.fit(model=module, train_dataloaders=train_dataloader) + self.assertIsNone(result) + + def test_load_from_new_checkpoint(self): + try: + DelightfulTTS.load_from_checkpoint( + checkpoint, + strict=False, + preprocess_config=self.preprocessing_config, + ) + except Exception as e: + self.fail(f"Loading from checkpoint raised an exception: {e}") + + def test_forward(self): + # device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + device = torch.device("cpu") + + module = DelightfulTTS.load_from_checkpoint( + checkpoint, + strict=False, + map_location=device, + preprocess_config=self.preprocessing_config, + ) + univnet = UnivNet() + univnet = univnet.to(device) + + text = """As the snake shook its head, a deafening shout behind Harry made both of them jump. + 'DUDLEY! MR DURSLEY! COME AND LOOK AT THIS SNAKE! YOU WON'T BELIEVE WHAT IT'S DOING!' + Dudley came waddling towards them as fast as he could. + ‘Out of the way, you,’ he said, punching Harry in the ribs. Caught by surprise, Harry fell hard on the concrete floor. What came next happened so fast no one saw how it happened – one second, Piers and Dudley were leaning right up close to the glass, the next, they had leapt back with howls of horror.""" + + speaker = torch.tensor([2071], device=device) + + mel_spec = module.forward( + text, + speaker, + ) + + self.assertIsInstance(mel_spec, torch.Tensor) diff --git a/models/tts/delightful_tts/train/config.yml b/models/tts/delightful_tts/train/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..13dd045365b6a2a95e05e0eff0adb2bbdddf3427 --- /dev/null +++ b/models/tts/delightful_tts/train/config.yml @@ -0,0 +1,45 @@ +seed_everything: true +trainer: + accelerator: auto + strategy: auto + devices: auto + num_nodes: 1 + precision: null + logger: null + callbacks: null + fast_dev_run: false + max_epochs: null + min_epochs: null + max_steps: -1 + min_steps: null + max_time: null + limit_train_batches: null + limit_val_batches: null + limit_test_batches: null + limit_predict_batches: null + overfit_batches: 0.0 + val_check_interval: null + check_val_every_n_epoch: 1 + num_sanity_val_steps: null + log_every_n_steps: null + enable_checkpointing: null + enable_progress_bar: null + enable_model_summary: null + accumulate_grad_batches: 1 + gradient_clip_val: null + gradient_clip_algorithm: null + deterministic: null + benchmark: null + inference_mode: true + use_distributed_sampler: true + profiler: null + detect_anomaly: false + barebones: false + plugins: null + sync_batchnorm: false + reload_dataloaders_every_n_epochs: 0 + default_root_dir: null +model: + out_dim: 10 + learning_rate: 0.02 +ckpt_path: null \ No newline at end of file diff --git a/models/tts/delightful_tts/train/run_save.sh b/models/tts/delightful_tts/train/run_save.sh new file mode 100644 index 0000000000000000000000000000000000000000..85f9c17a6b1eb3a88be2a07f038573366253676e --- /dev/null +++ b/models/tts/delightful_tts/train/run_save.sh @@ -0,0 +1,5 @@ +/home/oaiw/anaconda3/envs/tts_framework/bin/python train_dist.py +if [ $? -ne 0 ]; then + echo "An error occurred while running train_dist.py" + sudo shutdown -h now +fi \ No newline at end of file diff --git a/models/tts/delightful_tts/train/train.ipynb b/models/tts/delightful_tts/train/train.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f9cfb68f7d4b38837a99c0056f7aef14f2153c19 --- /dev/null +++ b/models/tts/delightful_tts/train/train.ipynb @@ -0,0 +1,203 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install env deps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "268d87b8-54a2-4e09-bbef-ed28f93719f3", + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install numba numpy scikit-learn tqdm pynini datasets deep-phonemizer nemo-text-processing piq soundfile transformers unidecode tensorboard librosa gpustat chardet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0662705-c8a6-4588-9435-07aecb004825", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install lightning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add tensorboard to track the basic metrics and outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "089094c9-4b03-47a8-a500-50f15d9a8ff7", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext tensorboard" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34c26745-47fd-4ad8-9bfc-d3952a406b2a", + "metadata": {}, + "outputs": [], + "source": [ + "default_root_dir=\"logs/acoustic\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de8ed1db-2d23-4f34-abad-43d287a2ffd9", + "metadata": {}, + "outputs": [], + "source": [ + "%tensorboard --logdir {default_root_dir}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d26af70-b7ea-4e64-888c-106a984a3fdd", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from lightning.pytorch import Trainer\n", + "from lightning.pytorch.callbacks import StochasticWeightAveraging\n", + "from lightning.pytorch.loggers import TensorBoardLogger\n", + "from lightning.pytorch.tuner.tuning import Tuner\n", + "\n", + "from training.modules import AcousticModule, VocoderModule, AcousticDataModule\n", + "\n", + "os.environ[\"CUDA_LAUNCH_BLOCKING\"]=\"1\"\n", + "CUDA_LAUNCH_BLOCKING=1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fdbeb42-0158-4c77-874b-6fd3f72e1219", + "metadata": {}, + "outputs": [], + "source": [ + "accelerator=\"cuda\"\n", + "ckpt_acoustic=\"./checkpoints/am_pitche_stats_with_vocoder.ckpt\"\n", + "ckpt_vocoder=\"./checkpoints/vocoder.ckpt\"\n", + "\n", + "# Control Validation Frequency\n", + "check_val_every_n_epoch=10\n", + "# Accumulate gradients\n", + "accumulate_grad_batches=5\n", + "# SWA learning rate\n", + "swa_lrs=1e-2\n", + "\n", + "# Stochastic Weight Averaging (SWA) can make your models generalize\n", + "# better at virtually no additional cost.\n", + "# This can be used with both non-trained and trained models.\n", + "# The SWA procedure smooths the loss landscape thus making it\n", + "# harder to end up in a local minimum during optimization.\n", + "callbacks = [\n", + " StochasticWeightAveraging(swa_lrs=swa_lrs),\n", + " # TODO: Add EarlyStopping Callback\n", + "]\n", + "\n", + "tensorboard = TensorBoardLogger(save_dir=default_root_dir)\n", + "\n", + "trainer = Trainer(\n", + " logger=tensorboard,\n", + " # Save checkpoints to the `default_root_dir` directory\n", + " default_root_dir=default_root_dir,\n", + " accelerator=accelerator,\n", + " check_val_every_n_epoch=check_val_every_n_epoch,\n", + " accumulate_grad_batches=accumulate_grad_batches,\n", + " max_epochs=-1,\n", + " callbacks=callbacks,\n", + " devices=2,\n", + ")\n", + "\n", + "# Load the pretrained weights for the vocoder\n", + "vocoder_module = VocoderModule.load_from_checkpoint(\n", + " ckpt_vocoder,\n", + ")\n", + "\n", + "module = AcousticModule.load_from_checkpoint(\n", + " ckpt_acoustic,\n", + " vocoder_module=vocoder_module,\n", + ")\n", + "\n", + "datamodule = AcousticDataModule(batch_size=module.train_config.batch_size)\n", + "\n", + "# Create a Tuner\n", + "tuner = Tuner(trainer)\n", + "\n", + "# finds learning rate automatically\n", + "# sets hparams.lr or hparams.learning_rate to that learning rate\n", + "tuner.lr_find(module)\n", + "\n", + "tuner.scale_batch_size(module, datamodule=datamodule)\n", + "\n", + "# vocoder_module = VocoderModule()\n", + "# module = AcousticModule()\n", + "\n", + "# train_dataloader = module.train_dataloader()\n", + "\n", + "trainer.fit(model=module) #, train_dataloaders=train_dataloader)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16370731-72bb-497a-a1cc-00b0a5d1496f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "environment": { + "kernel": "conda-env-tts_framework-py", + "name": "workbench-notebooks.m111", + "type": "gcloud", + "uri": "gcr.io/deeplearning-platform-release/workbench-notebooks:m111" + }, + "kernelspec": { + "display_name": "tts_framework", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/models/tts/delightful_tts/train/train.py b/models/tts/delightful_tts/train/train.py new file mode 100644 index 0000000000000000000000000000000000000000..336902d7bf824495513d61e3467b29179d51a7a8 --- /dev/null +++ b/models/tts/delightful_tts/train/train.py @@ -0,0 +1,102 @@ +from datetime import datetime +import logging +import os +import sys + +from lightning.pytorch import Trainer +from lightning.pytorch.accelerators import find_usable_cuda_devices # type: ignore +from lightning.pytorch.strategies import DDPStrategy +from lightning.pytorch.tuner.tuning import Tuner +import torch + +from models.tts.delightful_tts.delightful_tts import DelightfulTTS + +# Node runk in the cluster +node_rank = 0 +num_nodes = 2 + +# Setup of the training cluster +os.environ["MASTER_PORT"] = "12355" +# Change the IP address to the IP address of the master node +os.environ["MASTER_ADDR"] = "10.148.0.6" +os.environ["WORLD_SIZE"] = f"{num_nodes}" +# Change the IP address to the IP address of the master node +os.environ["NODE_RANK"] = f"{node_rank}" + +# Get the current date and time +now = datetime.now() + +# Format the current date and time as a string +timestamp = now.strftime("%Y%m%d_%H%M%S") + +# Create a logger +logger = logging.getLogger("my_logger") + +# Set the level of the logger to ERROR +logger.setLevel(logging.ERROR) + +# Create a file handler that logs error messages to a file with the current timestamp in its name +handler = logging.FileHandler(f"logs/error_{timestamp}.log") + +# Create a formatter and add it to the handler +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) + +# Add the handler to the logger +logger.addHandler(handler) + + +print("usable_cuda_devices: ", find_usable_cuda_devices()) + +# Set the precision of the matrix multiplication to float32 to improve the performance of the training +torch.set_float32_matmul_precision("high") + +default_root_dir = "logs" + +ckpt_acoustic = "./checkpoints/epoch=301-step=124630.ckpt" + +ckpt_vocoder = "./checkpoints/vocoder.ckpt" + +try: + trainer = Trainer( + accelerator="cuda", + devices=-1, + num_nodes=num_nodes, + strategy=DDPStrategy( + gradient_as_bucket_view=True, + find_unused_parameters=True, + ), + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + enable_checkpointing=True, + accumulate_grad_batches=5, + max_epochs=-1, + log_every_n_steps=10, + gradient_clip_val=0.5, + ) + + # model = DelightfulTTS() + model = DelightfulTTS.load_from_checkpoint(ckpt_acoustic, strict=False) + + tuner = Tuner(trainer) + tuner.lr_find(model) + # ValueError: Tuning the batch size is currently not supported with distributed strategies. + # tuner.scale_batch_size(model, mode="binsearch") + + train_dataloader = model.train_dataloader( + # NOTE: Preload the cached dataset into the RAM + cache_dir="/dev/shm/", + cache=True, + ) + + trainer.fit( + model=model, + train_dataloaders=train_dataloader, + # Resume training states from the checkpoint file + # ckpt_path=ckpt_acoustic, + ) + +except Exception as e: + # Log the error message + logger.error(f"An error occurred: {e}") + sys.exit(1) diff --git a/models/tts/delightful_tts/train/train_cli.py b/models/tts/delightful_tts/train/train_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..7107252d78de81a73c596a1ade2c20cd51f7d52d --- /dev/null +++ b/models/tts/delightful_tts/train/train_cli.py @@ -0,0 +1,94 @@ +from argparse import ArgumentParser + +from lightning.pytorch import Trainer +from lightning.pytorch.callbacks import StochasticWeightAveraging +from lightning.pytorch.loggers import TensorBoardLogger +from lightning.pytorch.tuner.tuning import Tuner + +from models.tts.delightful_tts import DelightfulTTS +from models.vocoder.univnet import UnivNet + + +def train(): + parser = ArgumentParser() + + # Trainer arguments + parser.add_argument("--devices", type=int, default=None) + parser.add_argument("--default_root_dir", type=str, default="logs/acoustic") + + parser.add_argument("--limit_train_batches", type=int, default=None) + parser.add_argument("--max_epochs", type=int, default=None) + + # Default we use 3 batches to accumulate gradients + parser.add_argument("--accumulate_grad_batches", type=int, default=3) + parser.add_argument("--accelerator", type=str, default="cuda") + parser.add_argument("--ckpt_acoustic", type=str, default="./checkpoints/am_pitche_stats_with_vocoder.ckpt") + parser.add_argument("--ckpt_vocoder", type=str, default="./checkpoints/vocoder.ckpt") + + # Optimizers + # FIXME: this is not working, found an errors... + # Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment + # Stochastic Weight Averaging (SWA) + parser.add_argument("--swa", type=bool, default=False) + # Learning rate finder + parser.add_argument("--lr_find", type=bool, default=False) + # Batch size scaling + parser.add_argument("--batch_size_scaling", type=bool, default=False) + + # Parse the user inputs and defaults (returns a argparse.Namespace) + args = parser.parse_args() + + tensorboard = TensorBoardLogger(save_dir=args.default_root_dir) + + callbacks = [] + + if args.swa: + callbacks.append( + # Stochastic Weight Averaging (SWA) can make your models generalize + # better at virtually no additional cost. + # This can be used with both non-trained and trained models. + # The SWA procedure smooths the loss landscape thus making it + # harder to end up in a local minimum during optimization. + StochasticWeightAveraging(swa_lrs=1e-2), + ) + + trainer = Trainer( + logger=tensorboard, + # Save checkpoints to the `default_root_dir` directory + default_root_dir=args.default_root_dir, + limit_train_batches=args.limit_train_batches, + max_epochs=args.max_epochs, + accelerator=args.accelerator, + accumulate_grad_batches=args.accumulate_grad_batches, + callbacks=callbacks, + ) + + # Create a Tuner + tuner = Tuner(trainer) + + # Load the pretrained weights for the vocoder + vocoder_module = UnivNet.load_from_checkpoint( + args.ckpt_vocoder, + ) + module = DelightfulTTS.load_from_checkpoint( + args.ckpt_acoustic, + vocoder_module=vocoder_module, + ) + + train_dataloader = module.train_dataloader() + + if args.lr_find: + # finds learning rate automatically + # sets hparams.lr or hparams.learning_rate to that learning rate + tuner.lr_find(module) + + if args.batch_size_scaling: + # Auto-scale batch size by growing it exponentially (default) + tuner.scale_batch_size(module, init_val=2, mode="power") + + trainer.fit(model=module, train_dataloaders=train_dataloader) + + +if __name__ == "__main__": + train() + # note: it is good practice to implement the CLI in a function and call it in the main if block diff --git a/models/tts/delightful_tts/train/train_new.py b/models/tts/delightful_tts/train/train_new.py new file mode 100644 index 0000000000000000000000000000000000000000..b8069a1cf974f1fadf233dabe63f001d8ce11ed0 --- /dev/null +++ b/models/tts/delightful_tts/train/train_new.py @@ -0,0 +1,93 @@ +from datetime import datetime +import logging +import os + +from lightning.pytorch import Trainer +from lightning.pytorch.accelerators import find_usable_cuda_devices # type: ignore +from lightning.pytorch.strategies import DDPStrategy +import torch + +from models.tts.delightful_tts.delightful_tts_refined import DelightfulTTS + +# Node runk in the cluster +node_rank = 0 +num_nodes = 4 + +# # Setup of the training cluster +os.environ["MASTER_PORT"] = "12355" +# # # Change the IP address to the IP address of the master node +os.environ["MASTER_ADDR"] = "10.164.0.32" +os.environ["WORLD_SIZE"] = f"{num_nodes}" +# # # Change the IP address to the IP address of the master node +os.environ["NODE_RANK"] = f"{node_rank}" + +# Get the current date and time +now = datetime.now() + +# Format the current date and time as a string +timestamp = now.strftime("%Y%m%d_%H%M%S") + +# Create a logger +logger = logging.getLogger("my_logger") + +# Set the level of the logger to ERROR +logger.setLevel(logging.ERROR) + +# Create a file handler that logs error messages to a file with the current timestamp in its name +handler = logging.FileHandler(f"logs/error_{timestamp}.log") + +# Create a formatter and add it to the handler +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) + +# Add the handler to the logger +logger.addHandler(handler) + +print("usable_cuda_devices: ", find_usable_cuda_devices()) + +# Set the precision of the matrix multiplication to float32 to improve the performance of the training +torch.set_float32_matmul_precision("high") + +# Root and checkpoint +default_root_dir = "logs_new3" +ckpt_acoustic = ( + "./logs_new3/lightning_logs/version_4/checkpoints/epoch=33-step=4046.ckpt" +) + +trainer = Trainer( + accelerator="cuda", + devices=-1, + num_nodes=num_nodes, + strategy=DDPStrategy( + gradient_as_bucket_view=True, + find_unused_parameters=True, + ), + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + enable_checkpointing=True, + accumulate_grad_batches=5, + max_epochs=-1, + log_every_n_steps=10, + gradient_clip_val=0.5, +) + +# model = DelightfulTTS() +# model = DelightfulTTS(batch_size=10) +model = DelightfulTTS.load_from_checkpoint(ckpt_acoustic, strict=False) + +train_dataloader = model.train_dataloader( + root="/dev/shm/", + # NOTE: Preload the cached dataset into the RAM + cache_dir="/dev/shm/", + cache=True, + include_libri=False, + libri_speakers=[], + hifi_speakers=["John Van Stan"], +) + +trainer.fit( + model=model, + train_dataloaders=train_dataloader, + # Resume training states from the checkpoint file + ckpt_path=ckpt_acoustic, +) diff --git a/models/vocoder/__init__.py b/models/vocoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5221ffecc06c2e9282e2d5f7686f55c1a5966038 --- /dev/null +++ b/models/vocoder/__init__.py @@ -0,0 +1 @@ +from .univnet import UnivNet diff --git a/models/vocoder/hifigan/__init__.py b/models/vocoder/hifigan/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1303c9dce93c3caad27d46bca0cde95bf481ce39 --- /dev/null +++ b/models/vocoder/hifigan/__init__.py @@ -0,0 +1 @@ +from .hifigan import HifiGan diff --git a/models/vocoder/hifigan/config_v1_48Khz_multiGPU.json b/models/vocoder/hifigan/config_v1_48Khz_multiGPU.json new file mode 100644 index 0000000000000000000000000000000000000000..9ac69eba0688246770f306782766a8d1fc35c616 --- /dev/null +++ b/models/vocoder/hifigan/config_v1_48Khz_multiGPU.json @@ -0,0 +1,38 @@ +{ + "resblock": "1", + "num_gpus": 3, + "batch_size": 8, + "learning_rate": 0.0002, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.9995, + "seed": 1234, + + "upsample_rates": [ 8, 8, 2, 2, 2], + "upsample_kernel_sizes": [16,16, 4, 4, 4], + "upsample_initial_channel": 512, + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + "discriminator_periods": [3, 5, 7, 11, 17, 23, 37], + + "segment_size": 16384, + "num_mels": 80, + "num_freq": 1025, + "n_fft" : 2048, + "hop_size": 512, + "win_size": 2048, + + "sampling_rate": 44100, + + "fmin": 20, + "fmax": 11025, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/vocoder/hifigan/generator.py b/models/vocoder/hifigan/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..d629cef77386424033f02205a1c5f0572a8ab05c --- /dev/null +++ b/models/vocoder/hifigan/generator.py @@ -0,0 +1,265 @@ +from typing import List, Tuple + +import torch +from torch import Tensor, nn +from torch.nn import Conv1d, ConvTranspose1d, Module +import torch.nn.functional as F +from torch.nn.utils import remove_weight_norm, weight_norm + +from models.config import HifiGanConfig, HifiGanPretrainingConfig, PreprocessingConfig + +from .utils import get_padding, init_weights + +# Leaky ReLU slope +LRELU_SLOPE = HifiGanPretrainingConfig.lReLU_slope + + +class ResBlock1(Module): + def __init__( + self, + channels: int, + kernel_size: int = 3, + dilation: List[int] = [1, 3, 5], + ): + r"""Initialize the ResBlock1 module. + + Args: + channels (int): The number of channels for the ResBlock. + kernel_size (int, optional): The kernel size for the convolutional layers. Defaults to 3. + dilation (Tuple[int, int, int], optional): The dilation for the convolutional layers. Defaults to (1, 3, 5). + """ + super().__init__() + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ), + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ), + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]), + ), + ), + ], + ) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ), + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ), + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ), + ), + ], + ) + self.convs2.apply(init_weights) + + def forward(self, x: Tensor) -> Tensor: + r"""Forward pass of the ResBlock1 module. + + Args: + x (Tensor): The input tensor. + + Returns: + Tensor: The output tensor. + """ + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, LRELU_SLOPE) + xt = c1(xt) + xt = F.leaky_relu(xt, LRELU_SLOPE) + xt = c2(xt) + x = xt + x + return x + + def remove_weight_norm(self): + r"""Remove the weight normalization from the convolutional layers.""" + for layer in self.convs1: + remove_weight_norm(layer) + for layer in self.convs2: + remove_weight_norm(layer) + + +class ResBlock2(Module): + def __init__( + self, + channels: int, + kernel_size: int = 3, + dilation: List[int] = [1, 3], + ): + r"""Initialize the ResBlock2 module. + + Args: + channels (int): The number of channels for the ResBlock. + kernel_size (int, optional): The kernel size for the convolutional layers. Defaults to 3. + dilation (Tuple[int, int], optional): The dilation for the convolutional layers. Defaults to (1, 3). + """ + super().__init__() + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ), + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ), + ), + ], + ) + self.convs.apply(init_weights) + + def forward(self, x: Tensor) -> Tensor: + r"""Forward pass of the ResBlock2 module. + + Args: + x (Tensor): The input tensor. + + Returns: + Tensor: The output tensor. + """ + for layer in self.convs: + xt = F.leaky_relu(x, LRELU_SLOPE) + xt = layer(xt) + x = xt + x + return x + + def remove_weight_norm(self): + r"""Remove the weight normalization from the convolutional layers.""" + for layer in self.convs: + remove_weight_norm(layer) + + +class Generator(Module): + def __init__(self, h: HifiGanConfig, p: PreprocessingConfig): + r"""Initialize the Generator module. + + Args: + h (HifiGanConfig): The configuration for the Generator. + p (PreprocessingConfig): The configuration for the preprocessing. + """ + super().__init__() + self.h = h + self.p = p + self.num_kernels = len(h.resblock_kernel_sizes) + self.num_upsamples = len(h.upsample_rates) + self.conv_pre = weight_norm( + Conv1d( + self.p.stft.n_mel_channels, + h.upsample_initial_channel, + 7, + 1, + padding=3, + ), + ) + resblock = ResBlock1 if h.resblock == "1" else ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + h.upsample_initial_channel // (2**i), + h.upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ), + ), + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + resblock_list = nn.ModuleList() + ch = h.upsample_initial_channel // (2 ** (i + 1)) + for _, (k, d) in enumerate( + zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes), + ): + resblock_list.append(resblock(ch, k, d)) + self.resblocks.append(resblock_list) + + self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3)) + self.ups.apply(init_weights) + self.conv_post.apply(init_weights) + + def forward(self, x: Tensor) -> Tensor: + r"""Forward pass of the Generator module. + + Args: + x (Tensor): The input tensor. + + Returns: + Tensor: The output tensor. + """ + x = self.conv_pre(x) + + for upsample_layer, resblock_group in zip(self.ups, self.resblocks): + x = F.leaky_relu(x, LRELU_SLOPE) + x = upsample_layer(x) + xs = torch.zeros(x.shape, dtype=x.dtype, device=x.device) + for resblock in resblock_group: # type: ignore + xs += resblock(x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x diff --git a/models/vocoder/hifigan/hifigan.py b/models/vocoder/hifigan/hifigan.py new file mode 100644 index 0000000000000000000000000000000000000000..fb374813cf64c9e02169d8fdf3e37fca9a6e9221 --- /dev/null +++ b/models/vocoder/hifigan/hifigan.py @@ -0,0 +1,304 @@ +import itertools +from typing import List + +from lightning.pytorch.core import LightningModule +import torch +from torch import nn +from torch.optim import AdamW, Optimizer +from torch.optim.lr_scheduler import ExponentialLR +from torch.utils.data import DataLoader + +from models.config import ( + HifiGanConfig, + HifiGanPretrainingConfig, +) +from models.config import ( + PreprocessingConfigHifiGAN as PreprocessingConfig, +) +from training.datasets.hifi_gan_dataset import train_dataloader +from training.loss import ( + DiscriminatorLoss, + FeatureMatchingLoss, + GeneratorLoss, +) +from training.preprocess import TacotronSTFT + +from .generator import Generator +from .mp_discriminator import MultiPeriodDiscriminator +from .ms_discriminator import MultiScaleDiscriminator + + +class HifiGan(LightningModule): + r"""HifiGan module. + + This module contains the `Generator` and `Discriminator` models, and handles training and optimization. + """ + + def __init__( + self, + lang: str = "en", + batch_size: int = 16, + sampling_rate: int = 44100, + ): + r"""Initializes the `HifiGan`. + + Args: + fine_tuning (bool, optional): Whether to use fine-tuning mode or not. Defaults to False. + lang (str): Language of the dataset. + batch_size (int): The batch size. Defaults to 16. + sampling_rate (int): The sampling rate of the audio. Defaults to 44100. + """ + super().__init__() + + self.batch_size = batch_size + self.sampling_rate = sampling_rate + self.lang = lang + + self.preprocess_config = PreprocessingConfig( + "multilingual", + sampling_rate=sampling_rate, + ) + self.train_config = HifiGanPretrainingConfig() + + self.generator = Generator( + h=HifiGanConfig(), + p=self.preprocess_config, + ) + self.mpd = MultiPeriodDiscriminator() + self.msd = MultiScaleDiscriminator() + + self.feature_loss = FeatureMatchingLoss() + self.discriminator_loss = DiscriminatorLoss() + self.generator_loss = GeneratorLoss() + self.mae_loss = nn.L1Loss() + + self.tacotronSTFT = TacotronSTFT( + filter_length=self.preprocess_config.stft.filter_length, + hop_length=self.preprocess_config.stft.hop_length, + win_length=self.preprocess_config.stft.win_length, + n_mel_channels=self.preprocess_config.stft.n_mel_channels, + sampling_rate=self.preprocess_config.sampling_rate, + mel_fmin=self.preprocess_config.stft.mel_fmin, + mel_fmax=self.preprocess_config.stft.mel_fmax, + center=False, + ) + + # Mark TacotronSTFT as non-trainable + for param in self.tacotronSTFT.parameters(): + param.requires_grad = False + + # Switch to manual optimization + self.automatic_optimization = False + + def forward(self, y_pred: torch.Tensor) -> torch.Tensor: + r"""Performs a forward pass through the UnivNet model. + + Args: + y_pred (torch.Tensor): The predicted mel spectrogram. + + Returns: + torch.Tensor: The output of the UnivNet model. + """ + wav_prediction = self.generator.forward(y_pred) + + return wav_prediction.squeeze() + + def training_step(self, batch: List, batch_idx: int): + r"""Performs a training step for the model. + + Args: + batch (Tuple[str, Tensor, Tensor]): The batch of data for training. Each item in the list is a tuple containing the ID of the item, the audio waveform, and the mel spectrogram. + batch_idx (int): Index of the batch. + + Returns: + dict: A dictionary containing the total loss for the generator and logs for tensorboard. + """ + _, audio, mel = batch + + # Access your optimizers + optimizers = self.optimizers() + schedulers = self.lr_schedulers() + opt_generator: Optimizer = optimizers[0] # type: ignore + sch_generator: ExponentialLR = schedulers[0] # type: ignore + + opt_discriminator: Optimizer = optimizers[1] # type: ignore + sch_discriminator: ExponentialLR = schedulers[1] # type: ignore + + # Generate fake audio + audio_pred = self.generator.forward(mel) + _, fake_mel = self.tacotronSTFT(audio_pred.squeeze(1)) + + # Train discriminator + opt_discriminator.zero_grad() + mpd_score_real, mpd_score_gen, _, _ = self.mpd.forward( + y=audio, + y_hat=audio_pred.detach(), + ) + loss_disc_mpd, _, _ = self.discriminator_loss.forward( + disc_real_outputs=mpd_score_real, + disc_generated_outputs=mpd_score_gen, + ) + msd_score_real, msd_score_gen, _, _ = self.msd( + y=audio, + y_hat=audio_pred.detach(), + ) + loss_disc_msd, _, _ = self.discriminator_loss( + disc_real_outputs=msd_score_real, + disc_generated_outputs=msd_score_gen, + ) + loss_d = loss_disc_msd + loss_disc_mpd + + # Step for the discriminator + self.manual_backward(loss_d, retain_graph=True) + opt_discriminator.step() + + # Train generator + opt_generator.zero_grad() + loss_mel = self.mae_loss(fake_mel, mel) + + _, mpd_score_gen, fmap_mpd_real, fmap_mpd_gen = self.mpd.forward( + y=audio, + y_hat=audio_pred, + ) + _, msd_score_gen, fmap_msd_real, fmap_msd_gen = self.msd.forward( + y=audio, + y_hat=audio_pred, + ) + loss_fm_mpd = self.feature_loss.forward( + fmap_r=fmap_mpd_real, + fmap_g=fmap_mpd_gen, + ) + loss_fm_msd = self.feature_loss.forward( + fmap_r=fmap_msd_real, + fmap_g=fmap_msd_gen, + ) + loss_gen_mpd, _ = self.generator_loss.forward(disc_outputs=mpd_score_gen) + loss_gen_msd, _ = self.generator_loss.forward(disc_outputs=msd_score_gen) + loss_g = ( + loss_gen_msd + + loss_gen_mpd + + loss_fm_msd + + loss_fm_mpd + + loss_mel * self.train_config.l1_factor + ) + + # step for the generator + self.manual_backward(loss_g, retain_graph=True) + opt_generator.step() + + # Schedulers step + sch_generator.step() + sch_discriminator.step() + + # Gen losses + self.log( + "loss_gen_msd", + loss_gen_msd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "loss_gen_mpd", + loss_gen_mpd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "loss_fm_msd", + loss_fm_msd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "loss_fm_mpd", + loss_fm_mpd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "mel_loss", + loss_mel, + sync_dist=True, + batch_size=self.batch_size, + ) + + # Disc logs + self.log( + "loss_disc_msd", + loss_disc_msd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "loss_disc_mpd", + loss_disc_mpd, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "total_loss_disc", + loss_d, + sync_dist=True, + batch_size=self.batch_size, + ) + + def configure_optimizers(self): + r"""Configures the optimizers and learning rate schedulers for the `UnivNet` and `Discriminator` models. + + This method creates an `AdamW` optimizer and an `ExponentialLR` scheduler for each model. + The learning rate, betas, and decay rate for the optimizers and schedulers are taken from the training configuration. + + Returns + tuple: A tuple containing two dictionaries. Each dictionary contains the optimizer and learning rate scheduler for one of the models. + """ + optim_generator = AdamW( + self.generator.parameters(), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_generator = ExponentialLR( + optim_generator, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + optim_discriminator = AdamW( + itertools.chain(self.msd.parameters(), self.mpd.parameters()), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_discriminator = ExponentialLR( + optim_discriminator, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + return ( + {"optimizer": optim_generator, "lr_scheduler": scheduler_generator}, + {"optimizer": optim_discriminator, "lr_scheduler": scheduler_discriminator}, + ) + + def train_dataloader( + self, + root: str = "datasets_cache", + cache: bool = True, + cache_dir: str = "/dev/shm", + ) -> DataLoader: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. Defaults to "/dev/shm". + + Returns: + Tupple[DataLoader, DataLoader]: The training and validation dataloaders. + """ + return train_dataloader( + batch_size=self.batch_size, + num_workers=self.preprocess_config.workers, + root=root, + cache=cache, + cache_dir=cache_dir, + lang=self.lang, + ) diff --git a/models/vocoder/hifigan/mp_discriminator.py b/models/vocoder/hifigan/mp_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..fbd0bf9a09a1aaac20ec4ebddf2f8338c2c996e8 --- /dev/null +++ b/models/vocoder/hifigan/mp_discriminator.py @@ -0,0 +1,157 @@ +from typing import List, Tuple + +import torch +from torch import Tensor, nn +from torch.nn import Conv2d, Module +import torch.nn.functional as F +from torch.nn.utils import spectral_norm, weight_norm + +from models.config import HifiGanPretrainingConfig + +from .utils import get_padding + +# Leaky ReLU slope +LRELU_SLOPE = HifiGanPretrainingConfig.lReLU_slope + + +class DiscriminatorP(Module): + def __init__( + self, + period: int, + kernel_size: int = 5, + stride: int = 3, + use_spectral_norm: bool = False, + ): + r"""Initialize the DiscriminatorP module. + + Args: + period (int): The period for the discriminator. + kernel_size (int, optional): The kernel size for the convolutional layers. Defaults to 5. + stride (int, optional): The stride for the convolutional layers. Defaults to 3. + use_spectral_norm (bool, optional): Whether to use spectral normalization. Defaults to False. + """ + super().__init__() + self.period = period + norm_f = weight_norm if not use_spectral_norm else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f( + Conv2d( + 1, + 32, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(5, 1), 0), + ), + ), + norm_f( + Conv2d( + 32, + 128, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(5, 1), 0), + ), + ), + norm_f( + Conv2d( + 128, + 512, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(5, 1), 0), + ), + ), + norm_f( + Conv2d( + 512, + 1024, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(5, 1), 0), + ), + ), + norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))), + ], + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x: Tensor) -> Tuple[Tensor, List[Tensor]]: + r"""Forward pass of the DiscriminatorP module. + + Args: + x (Tensor): The input tensor. + + Returns: + Tuple[Tensor, List[Tensor]]: The output tensor and a list of feature maps. + """ + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for layer in self.convs: + x = layer(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self): + r"""Initialize the MultiPeriodDiscriminator module.""" + super().__init__() + self.discriminators = nn.ModuleList( + [ + DiscriminatorP(2), + DiscriminatorP(3), + DiscriminatorP(5), + DiscriminatorP(7), + DiscriminatorP(11), + ], + ) + + def forward( + self, + y: Tensor, + y_hat: Tensor, + ) -> Tuple[ + List[torch.Tensor], + List[torch.Tensor], + List[torch.Tensor], + List[torch.Tensor], + ]: + r"""Forward pass of the MultiPeriodDiscriminator module. + + Args: + y (torch.Tensor): The real audio tensor. + y_hat (torch.Tensor): The generated audio tensor. + + Returns: + Tuple[List[torch.Tensor], List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]]: + A tuple containing lists of discriminator outputs and feature maps for real and generated audio. + """ + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + + for _, discriminator in enumerate(self.discriminators): + y_d_r, fmap_r = discriminator(y) + y_d_g, fmap_g = discriminator(y_hat) + + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs diff --git a/models/vocoder/hifigan/ms_discriminator.py b/models/vocoder/hifigan/ms_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..0eb489e36775f4ae6d4e7c8396933041a2ac0447 --- /dev/null +++ b/models/vocoder/hifigan/ms_discriminator.py @@ -0,0 +1,114 @@ +from typing import List, Tuple + +import torch +from torch import Tensor, nn +from torch.nn import AvgPool1d, Conv1d, Module +import torch.nn.functional as F +from torch.nn.utils import spectral_norm, weight_norm + +from models.config import HifiGanPretrainingConfig + +# Leaky ReLU slope +LRELU_SLOPE = HifiGanPretrainingConfig.lReLU_slope + + +class DiscriminatorS(Module): + def __init__(self, use_spectral_norm: bool = False): + r"""Initialize the DiscriminatorS module. + + Args: + use_spectral_norm (bool, optional): Whether to use spectral normalization. Defaults to False. + """ + super().__init__() + norm_f = weight_norm if not use_spectral_norm else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 128, 15, 1, padding=7)), + norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)), + norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)), + norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ], + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x: Tensor) -> Tuple[Tensor, List[Tensor]]: + r"""Forward pass of the DiscriminatorS module. + + Args: + x (Tensor): The input tensor. + + Returns: + Tuple[Tensor, List[Tensor]]: The output tensor and a list of feature maps. + """ + fmap = [] + for layer in self.convs: + x = layer(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiScaleDiscriminator(Module): + def __init__(self): + r"""Initialize the MultiScaleDiscriminator module.""" + super().__init__() + self.discriminators = nn.ModuleList( + [ + DiscriminatorS(use_spectral_norm=True), + DiscriminatorS(), + DiscriminatorS(), + ], + ) + self.meanpools = nn.ModuleList( + [ + AvgPool1d(4, 2, padding=2), + AvgPool1d(4, 2, padding=2), + ], + ) + + def forward( + self, + y: Tensor, + y_hat: Tensor, + ) -> Tuple[ + List[Tensor], + List[Tensor], + List[Tensor], + List[Tensor], + ]: + r"""Forward pass of the MultiScaleDiscriminator module. + + Args: + y (Tensor): The real audio tensor. + y_hat (Tensor): The generated audio tensor. + + Returns: + Tuple[List[Tensor], List[Tensor], List[Tensor], List[Tensor]]: + A tuple containing lists of discriminator outputs and feature maps for real and generated audio. + """ + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + + for i, discriminator in enumerate(self.discriminators): + if i != 0: + y = self.meanpools[i - 1](y) + y_hat = self.meanpools[i - 1](y_hat) + + y_d_r, fmap_r = discriminator(y) + y_d_g, fmap_g = discriminator(y_hat) + + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs diff --git a/models/vocoder/hifigan/tests/__init__.py b/models/vocoder/hifigan/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/vocoder/hifigan/tests/test_hifigan.py b/models/vocoder/hifigan/tests/test_hifigan.py new file mode 100644 index 0000000000000000000000000000000000000000..480e6b5c07d07316cf6e94be0884e524b6f935fb --- /dev/null +++ b/models/vocoder/hifigan/tests/test_hifigan.py @@ -0,0 +1,30 @@ +import unittest + +from lightning.pytorch import Trainer + +from models.vocoder.hifigan.hifigan import HifiGan + + +class TestHifiGan(unittest.TestCase): + def test_train_steps(self): + default_root_dir = "checkpoints" + + trainer = Trainer( + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + fast_dev_run=1, + limit_train_batches=1, + max_epochs=1, + accelerator="cpu", + ) + + module = HifiGan(batch_size=2) + + train_dataloader = module.train_dataloader(cache=False) + + result = trainer.fit(model=module, train_dataloaders=train_dataloader) + self.assertIsNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/hifigan/train.py b/models/vocoder/hifigan/train.py new file mode 100644 index 0000000000000000000000000000000000000000..989b74b870e36327d4bf80c8e28cbdb11ac16164 --- /dev/null +++ b/models/vocoder/hifigan/train.py @@ -0,0 +1,107 @@ +from datetime import datetime +import logging + +from lightning.pytorch import Trainer +from lightning.pytorch.accelerators import find_usable_cuda_devices # type: ignore +from lightning.pytorch.strategies import DDPStrategy +import torch + +from models.config import ( + HifiGanConfig, + PreprocessingConfig, +) +from models.vocoder.hifigan import HifiGan +from models.vocoder.hifigan.generator import Generator + +# Get the current date and time +now = datetime.now() + +# Format the current date and time as a string +timestamp = now.strftime("%Y%m%d_%H%M%S") + +# Create a logger +logger = logging.getLogger("my_logger") + +# Set the level of the logger to ERROR +logger.setLevel(logging.ERROR) + +# Create a file handler that logs error messages to a file with the current timestamp in its name +handler = logging.FileHandler(f"logs/error_{timestamp}.log") + +# Create a formatter and add it to the handler +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) + +# Add the handler to the logger +logger.addHandler(handler) + + +print("usable_cuda_devices: ", find_usable_cuda_devices()) + +# Set the precision of the matrix multiplication to float32 to improve the performance of the training +torch.set_float32_matmul_precision("high") + +default_root_dir = "logs_hifi_vocoder" + +# ckpt_acoustic="./checkpoints/epoch=301-step=124630.ckpt" + +# ckpt_vocoder="./checkpoints/vocoder.ckpt" + +trainer = Trainer( + accelerator="cuda", + devices=-1, + strategy=DDPStrategy( + gradient_as_bucket_view=True, + find_unused_parameters=True, + ), + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + enable_checkpointing=True, + max_epochs=-1, + log_every_n_steps=10, +) + +model = HifiGan() + +# Try to preload the model from the nvidia checkpoint +hifigan_state_dict = torch.load("./checkpoints/hifigan_spanish.pth") +model.load_state_dict(hifigan_state_dict) + +# Desc checkpoints state load +# disc_checkpoint_path = "checkpoints/do_02500000" +# disc_checkpoint = torch.load(disc_checkpoint_path) +# model.discriminator.MPD.load_state_dict(disc_checkpoint["mpd"]) +# model.discriminator.MSD.load_state_dict(disc_checkpoint["msd"]) + +# Gen checkpoints state load +# gen_checkpoint_path = "checkpoints/generator_v1" +# gen_checkpoint = torch.load(gen_checkpoint_path) +# model.generator.load_state_dict(gen_checkpoint["generator"]) + +# Reset the parameters of the generator +# preprocess_config = PreprocessingConfig( +# "multilingual", +# sampling_rate=44100, +# ) + +# model.generator = Generator( +# h=HifiGanConfig(), +# p=preprocess_config, +# ) + +# len(gen_checkpoint) + +train_dataloader = model.train_dataloader( + root="/dev/shm/", + # NOTE: Preload the cached dataset into the RAM + cache_dir="/dev/shm/", + cache=True, +) + +trainer.fit( + model=model, + train_dataloaders=train_dataloader, + # val_dataloaders=val_dataloader, + # Resume training states from the checkpoint file + # ckpt_path=ckpt_acoustic, +) diff --git a/models/vocoder/hifigan/utils.py b/models/vocoder/hifigan/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7c48d65f9edba5db8013ecc80ec315e5014e812d --- /dev/null +++ b/models/vocoder/hifigan/utils.py @@ -0,0 +1,17 @@ +from torch.nn.utils import weight_norm + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def apply_weight_norm(m): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + weight_norm(m) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) diff --git a/models/vocoder/univnet/__init__.py b/models/vocoder/univnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e1f79609f958180af9985265bcf21c2645a22aac --- /dev/null +++ b/models/vocoder/univnet/__init__.py @@ -0,0 +1,10 @@ +from .discriminator import Discriminator +from .discriminator_p import DiscriminatorP +from .discriminator_r import DiscriminatorR +from .generator import Generator +from .kernel_predictor import KernelPredictor +from .lvc_block import LVCBlock +from .multi_period_discriminator import MultiPeriodDiscriminator +from .multi_resolution_discriminator import MultiResolutionDiscriminator +from .traced_generator import TracedGenerator +from .univnet import UnivNet diff --git a/models/vocoder/univnet/discriminator.py b/models/vocoder/univnet/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..657de0902e69eb5f232918505c9d09e1a417faff --- /dev/null +++ b/models/vocoder/univnet/discriminator.py @@ -0,0 +1,41 @@ +import torch +from torch.nn import Module + +from models.config import VocoderModelConfig + +from .multi_period_discriminator import MultiPeriodDiscriminator +from .multi_resolution_discriminator import MultiResolutionDiscriminator + + +class Discriminator(Module): + r"""Discriminator for the UnuvNet vocoder. + + This class implements a discriminator that consists of a `MultiResolutionDiscriminator` and a `MultiPeriodDiscriminator`. + + Args: + model_config (VocoderModelConfig): Model configuration object. + + Attributes: + MRD (MultiResolutionDiscriminator): Multi-resolution discriminator instance. + MPD (MultiPeriodDiscriminator): Multi-period discriminator instance. + + Methods: + forward(x): Computes the forward pass of the discriminator. + + """ + + def __init__(self, model_config: VocoderModelConfig): + super().__init__() + self.MRD = MultiResolutionDiscriminator(model_config=model_config) + self.MPD = MultiPeriodDiscriminator(model_config=model_config) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Computes the forward pass of the discriminator. + + Args: + x (torch.Tensor): Input tensor of shape [B, C, T]. + + Returns: + tuple(torch.Tensor, torch.Tensor): Tuple containing the output tensors of the `MultiResolutionDiscriminator` and `MultiPeriodDiscriminator` instances. + """ + return self.MRD(x), self.MPD(x) diff --git a/models/vocoder/univnet/discriminator_p.py b/models/vocoder/univnet/discriminator_p.py new file mode 100644 index 0000000000000000000000000000000000000000..94f6f3c9007afb9f6859450291443e3ebc6404dc --- /dev/null +++ b/models/vocoder/univnet/discriminator_p.py @@ -0,0 +1,123 @@ +from typing import Any, Tuple + +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F +from torch.nn.utils.parametrizations import spectral_norm, weight_norm + +from models.config import VocoderModelConfig + + +class DiscriminatorP(Module): + r"""DiscriminatorP is a class that implements a discriminator network for the UnivNet vocoder. + + Args: + period (int): The period of the Mel spectrogram. + model_config (VocoderModelConfig): The configuration object for the UnivNet vocoder model. + """ + + def __init__( + self, + period: int, + model_config: VocoderModelConfig, + ): + super().__init__() + + self.LRELU_SLOPE = model_config.mpd.lReLU_slope + self.period = period + + kernel_size = model_config.mpd.kernel_size + stride = model_config.mpd.stride + + norm_f: Any = ( + spectral_norm if model_config.mpd.use_spectral_norm else weight_norm + ) + + self.convs = nn.ModuleList( + [ + norm_f( + nn.Conv2d( + 1, + 64, + (kernel_size, 1), + (stride, 1), + padding=(kernel_size // 2, 0), + ), + ), + norm_f( + nn.Conv2d( + 64, + 128, + (kernel_size, 1), + (stride, 1), + padding=(kernel_size // 2, 0), + ), + ), + norm_f( + nn.Conv2d( + 128, + 256, + (kernel_size, 1), + (stride, 1), + padding=(kernel_size // 2, 0), + ), + ), + norm_f( + nn.Conv2d( + 256, + 512, + (kernel_size, 1), + (stride, 1), + padding=(kernel_size // 2, 0), + ), + ), + norm_f( + nn.Conv2d( + 512, + 1024, + (kernel_size, 1), + 1, + padding=(kernel_size // 2, 0), + ), + ), + ], + ) + self.conv_post = norm_f( + nn.Conv2d( + 1024, + 1, + (3, 1), + 1, + padding=(1, 0), + ), + ) + + def forward(self, x: torch.Tensor) -> Tuple[list, torch.Tensor]: + r"""Forward pass of the discriminator network. + + Args: + x (torch.Tensor): The input tensor of shape (batch_size, channels, time_steps). + + Returns: + Tuple[list, torch.Tensor]: A tuple containing a list of feature maps and the output tensor of shape (batch_size, period). + """ + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for layers in self.convs: + x = layers(x.to(dtype=self.conv_post.weight.dtype)) + x = F.leaky_relu(x, self.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return fmap, x diff --git a/models/vocoder/univnet/discriminator_r.py b/models/vocoder/univnet/discriminator_r.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f7bd7b206ad5165e526fa1edf5a158f779d95c --- /dev/null +++ b/models/vocoder/univnet/discriminator_r.py @@ -0,0 +1,159 @@ +from typing import Any, Tuple + +import torch +from torch import nn +from torch.nn import Module +import torch.nn.functional as F +from torch.nn.utils.parametrizations import spectral_norm, weight_norm + +from models.config import VocoderModelConfig + + +class DiscriminatorR(Module): + r"""A class representing the Residual Discriminator network for a UnivNet vocoder. + + Args: + resolution (Tuple): A tuple containing the number of FFT points, hop length, and window length. + model_config (VocoderModelConfig): A configuration object for the UnivNet model. + """ + + def __init__( + self, + resolution: Tuple[int, int, int], + model_config: VocoderModelConfig, + ): + super().__init__() + + self.resolution = resolution + self.LRELU_SLOPE = model_config.mrd.lReLU_slope + + # Use spectral normalization or weight normalization based on the configuration + norm_f: Any = ( + spectral_norm if model_config.mrd.use_spectral_norm else weight_norm + ) + + # Define the convolutional layers + self.convs = nn.ModuleList( + [ + norm_f( + nn.Conv2d( + 1, + 32, + (3, 9), + padding=(1, 4), + ), + ), + norm_f( + nn.Conv2d( + 32, + 32, + (3, 9), + stride=(1, 2), + padding=(1, 4), + ), + ), + norm_f( + nn.Conv2d( + 32, + 32, + (3, 9), + stride=(1, 2), + padding=(1, 4), + ), + ), + norm_f( + nn.Conv2d( + 32, + 32, + (3, 9), + stride=(1, 2), + padding=(1, 4), + ), + ), + norm_f( + nn.Conv2d( + 32, + 32, + (3, 3), + padding=(1, 1), + ), + ), + ], + ) + self.conv_post = norm_f( + nn.Conv2d( + 32, + 1, + (3, 3), + padding=(1, 1), + ), + ) + + def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], torch.Tensor]: + r"""Forward pass of the DiscriminatorR class. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + tuple: A tuple containing the intermediate feature maps and the output tensor. + """ + fmap = [] + + # Compute the magnitude spectrogram of the input waveform + x = self.spectrogram(x) + + # Add a channel dimension to the spectrogram tensor + x = x.unsqueeze(1) + + # Apply the convolutional layers with leaky ReLU activation + for layer in self.convs: + x = layer(x.to(dtype=self.conv_post.weight.dtype)) + x = F.leaky_relu(x, self.LRELU_SLOPE) + fmap.append(x) + + # Apply the post-convolutional layer + x = self.conv_post(x) + fmap.append(x) + + # Flatten the output tensor + x = torch.flatten(x, 1, -1) + + return fmap, x + + def spectrogram(self, x: torch.Tensor) -> torch.Tensor: + r"""Computes the magnitude spectrogram of the input waveform. + + Args: + x (torch.Tensor): Input waveform tensor of shape [B, C, T]. + + Returns: + torch.Tensor: Magnitude spectrogram tensor of shape [B, F, TT], where F is the number of frequency bins and TT is the number of time frames. + """ + n_fft, hop_length, win_length = self.resolution + + # Apply reflection padding to the input waveform + x = F.pad( + x, + (int((n_fft - hop_length) / 2), int((n_fft - hop_length) / 2)), + mode="reflect", + ) + + # Squeeze the input waveform to remove the channel dimension + x = x.squeeze(1) + + # Compute the short-time Fourier transform of the input waveform + x = torch.stft( + x, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + center=False, + return_complex=True, + window=torch.ones(win_length, device=x.device), + ) # [B, F, TT, 2] + + x = torch.view_as_real(x) + + # Compute the magnitude spectrogram from the complex spectrogram + return torch.norm(x, p=2, dim=-1) # [B, F, TT] diff --git a/models/vocoder/univnet/generator.py b/models/vocoder/univnet/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..3c7ee82de7aee21e3010738774e45e39c1246f33 --- /dev/null +++ b/models/vocoder/univnet/generator.py @@ -0,0 +1,145 @@ +import torch +from torch import nn +from torch.nn import Module +from torch.nn.utils import parametrize + +from models.config import PreprocessingConfig, VocoderModelConfig +from models.helpers import get_mask_from_lengths + +from .lvc_block import LVCBlock + + +class Generator(Module): + """UnivNet Generator""" + + def __init__( + self, + model_config: VocoderModelConfig, + preprocess_config: PreprocessingConfig, + ): + r"""UnivNet Generator. + Initializes the UnivNet module. + + Args: + model_config (VocoderModelConfig): the model configuration. + preprocess_config (PreprocessingConfig): the preprocessing configuration. + """ + super().__init__() + + self.mel_channel = preprocess_config.stft.n_mel_channels + self.noise_dim = model_config.gen.noise_dim + self.hop_length = preprocess_config.stft.hop_length + channel_size = model_config.gen.channel_size + kpnet_conv_size = model_config.gen.kpnet_conv_size + + hop_length = 1 + self.res_stack = nn.ModuleList() + + for stride in model_config.gen.strides: + hop_length = stride * hop_length + self.res_stack.append( + LVCBlock( + channel_size, + preprocess_config.stft.n_mel_channels, + stride=stride, + dilations=model_config.gen.dilations, + lReLU_slope=model_config.gen.lReLU_slope, + cond_hop_length=hop_length, + kpnet_conv_size=kpnet_conv_size, + ), + ) + + self.conv_pre = nn.utils.parametrizations.weight_norm( + nn.Conv1d( + model_config.gen.noise_dim, + channel_size, + 7, + padding=3, + padding_mode="reflect", + ), + ) + + self.conv_post = nn.Sequential( + nn.LeakyReLU(model_config.gen.lReLU_slope), + nn.utils.parametrizations.weight_norm( + nn.Conv1d( + channel_size, + 1, + 7, + padding=3, + padding_mode="reflect", + ), + ), + nn.Tanh(), + ) + + # Output of STFT(zeros) + self.mel_mask_value = -11.5129 + + def forward(self, c: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the Generator module. + + Args: + c (Tensor): the conditioning sequence of mel-spectrogram (batch, mel_channels, in_length) + + Returns: + Tensor: the generated audio waveform (batch, 1, out_length) + """ + z = torch.randn( + c.shape[0], + self.noise_dim, + c.shape[2], + device=c.device, + dtype=self.conv_pre.weight.data.dtype, + ) + z = self.conv_pre(z) # (B, c_g, L) + + for res_block in self.res_stack: + z = res_block(z, c) # (B, c_g, L * s_0 * ... * s_i) + + return self.conv_post(z) # (B, 1, L * 256) + + def eval(self, inference: bool = False): + r"""Sets the module to evaluation mode. + + Args: + inference (bool): whether to remove weight normalization or not. + """ + super().eval() + # don't remove weight norm while validation in training loop + if inference: + self.remove_weight_norm() + + def remove_weight_norm(self) -> None: + r"""Removes weight normalization from the module.""" + print("Removing weight norm...") + + parametrize.remove_parametrizations(self.conv_pre, "weight") + + for layer in self.conv_post: + if len(layer.state_dict()) != 0: + parametrize.remove_parametrizations(layer, "weight") + + for res_block in self.res_stack: + res_block.remove_weight_norm() + + def infer(self, c: torch.Tensor, mel_lens: torch.Tensor) -> torch.Tensor: + r"""Infers the audio waveform from the mel-spectrogram conditioning sequence. + + Args: + c (Tensor): the conditioning sequence of mel-spectrogram (batch, mel_channels, in_length) + mel_lens (Tensor): the lengths of the mel-spectrogram conditioning sequence. + + Returns: + Tensor: the generated audio waveform (batch, 1, out_length) + """ + mel_mask = get_mask_from_lengths(mel_lens).unsqueeze(1).to(c.device) + c = c.masked_fill(mel_mask, self.mel_mask_value) + zero = torch.full( + (c.shape[0], self.mel_channel, 10), self.mel_mask_value, device=c.device, + ) + mel = torch.cat((c, zero), dim=2) + audio = self(mel) + audio = audio[:, :, : -(self.hop_length * 10)] + audio_mask = get_mask_from_lengths(mel_lens * 256).unsqueeze(1) + return audio.masked_fill(audio_mask, 0.0) diff --git a/models/vocoder/univnet/kernel_predictor.py b/models/vocoder/univnet/kernel_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..1d0254b33dab3b42ef4a9ebf70ca69ddb7f2fcf0 --- /dev/null +++ b/models/vocoder/univnet/kernel_predictor.py @@ -0,0 +1,151 @@ +import torch +from torch import nn +from torch.nn import Module +from torch.nn.utils import parametrize + + +class KernelPredictor(Module): + def __init__( + self, + cond_channels: int, + conv_in_channels: int, + conv_out_channels: int, + conv_layers: int, + conv_kernel_size: int = 3, + kpnet_hidden_channels: int = 64, + kpnet_conv_size: int = 3, + kpnet_dropout: float = 0.0, + lReLU_slope: float = 0.1, + ): + r"""Initializes a KernelPredictor object. + KernelPredictor is a class that predicts the kernel size for the convolutional layers in the UnivNet model. + The kernels of the LVC layers are predicted using a kernel predictor that takes the log-mel-spectrogram as the input. + + Args: + cond_channels (int): The number of channels for the conditioning sequence. + conv_in_channels (int): The number of channels for the input sequence. + conv_out_channels (int): The number of channels for the output sequence. + conv_layers (int): The number of layers in the model. + conv_kernel_size (int, optional): The kernel size for the convolutional layers. Defaults to 3. + kpnet_hidden_channels (int, optional): The number of hidden channels in the kernel predictor network. Defaults to 64. + kpnet_conv_size (int, optional): The kernel size for the kernel predictor network. Defaults to 3. + kpnet_dropout (float, optional): The dropout rate for the kernel predictor network. Defaults to 0.0. + lReLU_slope (float, optional): The slope for the leaky ReLU activation function. Defaults to 0.1. + """ + super().__init__() + + self.conv_in_channels = conv_in_channels + self.conv_out_channels = conv_out_channels + self.conv_kernel_size = conv_kernel_size + self.conv_layers = conv_layers + + kpnet_kernel_channels = ( + conv_in_channels * conv_out_channels * conv_kernel_size * conv_layers + ) # l_w + + kpnet_bias_channels = conv_out_channels * conv_layers # l_b + + padding = (kpnet_conv_size - 1) // 2 + + self.input_conv = nn.Sequential( + nn.utils.parametrizations.weight_norm( + nn.Conv1d( + cond_channels, + kpnet_hidden_channels, + 5, + padding=2, + bias=True, + ), + ), + nn.LeakyReLU(lReLU_slope), + ) + + self.residual_convs = nn.ModuleList( + [ + nn.Sequential( + nn.Dropout(kpnet_dropout), + nn.utils.parametrizations.weight_norm( + nn.Conv1d( + kpnet_hidden_channels, + kpnet_hidden_channels, + kpnet_conv_size, + padding=padding, + bias=True, + ), + ), + nn.LeakyReLU(lReLU_slope), + nn.utils.parametrizations.weight_norm( + nn.Conv1d( + kpnet_hidden_channels, + kpnet_hidden_channels, + kpnet_conv_size, + padding=padding, + bias=True, + ), + ), + nn.LeakyReLU(lReLU_slope), + ) + for _ in range(3) + ], + ) + + self.kernel_conv = nn.utils.parametrizations.weight_norm( + nn.Conv1d( + kpnet_hidden_channels, + kpnet_kernel_channels, + kpnet_conv_size, + padding=padding, + bias=True, + ), + ) + self.bias_conv = nn.utils.parametrizations.weight_norm( + nn.Conv1d( + kpnet_hidden_channels, + kpnet_bias_channels, + kpnet_conv_size, + padding=padding, + bias=True, + ), + ) + + def forward(self, c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Computes the forward pass of the model. + + Args: + c (Tensor): The conditioning sequence (batch, cond_channels, cond_length). + + Returns: + Tuple[Tensor, Tensor]: A tuple containing the kernel and bias tensors. + """ + batch, _, cond_length = c.shape + c = self.input_conv(c.to(dtype=self.kernel_conv.weight.dtype)) + for residual_conv in self.residual_convs: + c = c + residual_conv(c) + k = self.kernel_conv(c) + b = self.bias_conv(c) + kernels = k.contiguous().view( + batch, + self.conv_layers, + self.conv_in_channels, + self.conv_out_channels, + self.conv_kernel_size, + cond_length, + ) + bias = b.contiguous().view( + batch, + self.conv_layers, + self.conv_out_channels, + cond_length, + ) + + return kernels, bias + + def remove_weight_norm(self): + r"""Removes weight normalization from the input, kernel, bias, and residual convolutional layers.""" + parametrize.remove_parametrizations(self.input_conv[0], "weight") + parametrize.remove_parametrizations(self.kernel_conv, "weight") + parametrize.remove_parametrizations(self.bias_conv, "weight") + + for block in self.residual_convs: + parametrize.remove_parametrizations(block[1], "weight") # type: ignore + parametrize.remove_parametrizations(block[3], "weight") # type: ignore diff --git a/models/vocoder/univnet/lvc_block.py b/models/vocoder/univnet/lvc_block.py new file mode 100644 index 0000000000000000000000000000000000000000..23e1ea8ef01dcf7ce5abb6ec46a8e60a986c6e1c --- /dev/null +++ b/models/vocoder/univnet/lvc_block.py @@ -0,0 +1,205 @@ +from typing import List + +import torch +from torch import nn +from torch.nn import Module +from torch.nn import functional as F +from torch.nn.utils import parametrize + +from .kernel_predictor import KernelPredictor + + +class LVCBlock(Module): + r"""The location-variable convolutions block. + + To efficiently capture the local information of the condition, location-variable convolution (LVC) + obtained better sound quality and speed while maintaining the model size. + The kernels of the LVC layers are predicted using a kernel predictor that takes the log-mel-spectrogram + as the input. The kernel predictor is connected to a residual stack. One kernel predictor simultaneously + predicts the kernels of all LVC layers in one residual stack. + + Args: + in_channels (int): The number of input channels. + cond_channels (int): The number of conditioning channels. + stride (int): The stride of the convolutional layers. + dilations (List[int]): A list of dilation values for the convolutional layers. + lReLU_slope (float): The slope of the LeakyReLU activation function. + conv_kernel_size (int): The kernel size of the convolutional layers. + cond_hop_length (int): The hop length of the conditioning sequence. + kpnet_hidden_channels (int): The number of hidden channels in the kernel predictor network. + kpnet_conv_size (int): The kernel size of the convolutional layers in the kernel predictor network. + kpnet_dropout (float): The dropout rate for the kernel predictor network. + + Attributes: + cond_hop_length (int): The hop length of the conditioning sequence. + conv_layers (int): The number of convolutional layers. + conv_kernel_size (int): The kernel size of the convolutional layers. + kernel_predictor (KernelPredictor): The kernel predictor network. + convt_pre (nn.Sequential): The convolutional transpose layer. + conv_blocks (nn.ModuleList): The list of convolutional blocks. + + """ + + def __init__( + self, + in_channels: int, + cond_channels: int, + stride: int, + dilations: List[int] = [1, 3, 9, 27], + lReLU_slope: float = 0.2, + conv_kernel_size: int = 3, + cond_hop_length: int = 256, + kpnet_hidden_channels: int = 64, + kpnet_conv_size: int = 3, + kpnet_dropout: float = 0.0, + ): + super().__init__() + + self.cond_hop_length = cond_hop_length + self.conv_layers = len(dilations) + self.conv_kernel_size = conv_kernel_size + + self.kernel_predictor = KernelPredictor( + cond_channels=cond_channels, + conv_in_channels=in_channels, + conv_out_channels=2 * in_channels, + conv_layers=len(dilations), + conv_kernel_size=conv_kernel_size, + kpnet_hidden_channels=kpnet_hidden_channels, + kpnet_conv_size=kpnet_conv_size, + kpnet_dropout=kpnet_dropout, + lReLU_slope=lReLU_slope, + ) + + self.convt_pre = nn.Sequential( + nn.LeakyReLU(lReLU_slope), + nn.utils.parametrizations.weight_norm( + nn.ConvTranspose1d( + in_channels, + in_channels, + 2 * stride, + stride=stride, + padding=stride // 2 + stride % 2, + output_padding=stride % 2, + ), + ), + ) + + self.conv_blocks = nn.ModuleList( + [ + nn.Sequential( + nn.LeakyReLU(lReLU_slope), + nn.utils.parametrizations.weight_norm( + nn.Conv1d( + in_channels, + in_channels, + conv_kernel_size, + padding=dilation * (conv_kernel_size - 1) // 2, + dilation=dilation, + ), + ), + nn.LeakyReLU(lReLU_slope), + ) + for dilation in dilations + ], + ) + + def forward(self, x: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + r"""Forward propagation of the location-variable convolutions. + + Args: + x (Tensor): The input sequence (batch, in_channels, in_length). + c (Tensor): The conditioning sequence (batch, cond_channels, cond_length). + + Returns: + Tensor: The output sequence (batch, in_channels, in_length). + """ + _, in_channels, _ = x.shape # (B, c_g, L') + + x = self.convt_pre(x) # (B, c_g, stride * L') + kernels, bias = self.kernel_predictor(c) + + for i, conv in enumerate(self.conv_blocks): + output = conv(x) # (B, c_g, stride * L') + + k = kernels[:, i, :, :, :, :] # (B, 2 * c_g, c_g, kernel_size, cond_length) + b = bias[:, i, :, :] # (B, 2 * c_g, cond_length) + + output = self.location_variable_convolution( + output, k, b, hop_size=self.cond_hop_length, + ) # (B, 2 * c_g, stride * L'): LVC + x = x + torch.sigmoid(output[:, :in_channels, :]) * torch.tanh( + output[:, in_channels:, :], + ) # (B, c_g, stride * L'): GAU + + return x + + def location_variable_convolution( + self, + x: torch.Tensor, + kernel: torch.Tensor, + bias: torch.Tensor, + dilation: int = 1, + hop_size: int = 256, + ) -> torch.Tensor: + r"""Perform location-variable convolution operation on the input sequence (x) using the local convolution kernel. + Time: 414 μs ± 309 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each), test on NVIDIA V100. + + Args: + x (Tensor): The input sequence (batch, in_channels, in_length). + kernel (Tensor): The local convolution kernel (batch, in_channel, out_channels, kernel_size, kernel_length). + bias (Tensor): The bias for the local convolution (batch, out_channels, kernel_length). + dilation (int): The dilation of convolution. + hop_size (int): The hop_size of the conditioning sequence. + + Returns: + (Tensor): The output sequence after performing local convolution. (batch, out_channels, in_length). + """ + batch, _, in_length = x.shape + batch, _, out_channels, kernel_size, kernel_length = kernel.shape + assert in_length == ( + kernel_length * hop_size + ), "length of (x, kernel) is not matched" + + padding = dilation * int((kernel_size - 1) / 2) + x = F.pad( + x, (padding, padding), "constant", 0, + ) # (batch, in_channels, in_length + 2*padding) + x = x.unfold( + 2, hop_size + 2 * padding, hop_size, + ) # (batch, in_channels, kernel_length, hop_size + 2*padding) + + if hop_size < dilation: + x = F.pad(x, (0, dilation), "constant", 0) + x = x.unfold( + 3, dilation, dilation, + ) # (batch, in_channels, kernel_length, (hop_size + 2*padding)/dilation, dilation) + x = x[:, :, :, :, :hop_size] + x = x.transpose( + 3, 4, + ) # (batch, in_channels, kernel_length, dilation, (hop_size + 2*padding)/dilation) + x = x.unfold( + 4, kernel_size, 1, + ) # (batch, in_channels, kernel_length, dilation, _, kernel_size) + + o = torch.einsum("bildsk,biokl->bolsd", x, kernel) + o = o.contiguous(memory_format=torch.channels_last_3d) + + bias = ( + bias.unsqueeze(-1) + .unsqueeze(-1) + .contiguous(memory_format=torch.channels_last_3d) + ) + + o = o + bias + return o.contiguous().view(batch, out_channels, -1) + + def remove_weight_norm(self) -> None: + r"""Remove weight normalization from the convolutional layers in the LVCBlock. + + This method removes weight normalization from the kernel predictor and all convolutional layers in the LVCBlock. + """ + self.kernel_predictor.remove_weight_norm() + parametrize.remove_parametrizations(self.convt_pre[1], "weight") + for block in self.conv_blocks: + parametrize.remove_parametrizations(block[1], "weight") # type: ignore diff --git a/models/vocoder/univnet/multi_period_discriminator.py b/models/vocoder/univnet/multi_period_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..d6f7ebef65594951fd79063cd6ecee8e1d416cce --- /dev/null +++ b/models/vocoder/univnet/multi_period_discriminator.py @@ -0,0 +1,39 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import VocoderModelConfig + +from .discriminator_p import DiscriminatorP + + +class MultiPeriodDiscriminator(Module): + r"""MultiPeriodDiscriminator is a class that implements a multi-period discriminator network for the UnivNet vocoder. + + Args: + model_config (VocoderModelConfig): The configuration object for the UnivNet vocoder model. + """ + + def __init__( + self, + model_config: VocoderModelConfig, + ): + super().__init__() + + self.discriminators = nn.ModuleList( + [ + DiscriminatorP(period, model_config=model_config) + for period in model_config.mpd.periods + ], + ) + + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: + r"""Forward pass of the multi-period discriminator network. + + Args: + x (torch.Tensor): The input tensor of shape (batch_size, channels, time_steps). + + Returns: + list: A list of output tensors from each discriminator network. + """ + return [disc(x) for disc in self.discriminators] diff --git a/models/vocoder/univnet/multi_resolution_discriminator.py b/models/vocoder/univnet/multi_resolution_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..582021ed4f91c1edff24970905eca1294dd0ea59 --- /dev/null +++ b/models/vocoder/univnet/multi_resolution_discriminator.py @@ -0,0 +1,50 @@ +import torch +from torch import nn +from torch.nn import Module + +from models.config import VocoderModelConfig + +from .discriminator_r import DiscriminatorR + + +class MultiResolutionDiscriminator(Module): + r"""Multi-resolution discriminator for the UnivNet vocoder. + + This class implements a multi-resolution discriminator that consists of multiple DiscriminatorR instances, each operating at a different resolution. + + Args: + model_config (VocoderModelConfig): Model configuration object. + + Attributes: + resolutions (list): List of resolutions for each DiscriminatorR instance. + discriminators (nn.ModuleList): List of DiscriminatorR instances. + + Methods: + forward(x): Computes the forward pass of the multi-resolution discriminator. + + """ + + def __init__( + self, + model_config: VocoderModelConfig, + ): + super().__init__() + + self.resolutions = model_config.mrd.resolutions + self.discriminators = nn.ModuleList( + [ + DiscriminatorR(resolution, model_config=model_config) + for resolution in self.resolutions + ], + ) + + def forward(self, x: torch.Tensor) -> list[tuple[torch.Tensor, torch.Tensor]]: + r"""Computes the forward pass of the multi-resolution discriminator. + + Args: + x (torch.Tensor): Input tensor of shape [B, C, T]. + + Returns: + list: List of tuples containing the intermediate feature maps and the output scores for each `DiscriminatorR` instance. + """ + return [disc(x) for disc in self.discriminators] # [(feat, score), (feat, score), (feat, score)] diff --git a/models/vocoder/univnet/tests/__init__.py b/models/vocoder/univnet/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/models/vocoder/univnet/tests/test_discriminator.py b/models/vocoder/univnet/tests/test_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..810dafde63d0e9827d17e345206275b2f0712ec0 --- /dev/null +++ b/models/vocoder/univnet/tests/test_discriminator.py @@ -0,0 +1,144 @@ +import unittest + +import torch + +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.config import VocoderModelConfig +from models.vocoder.univnet import Discriminator, Generator + + +# One of the most important test case for univnet +# Integration test +class TestDiscriminator(unittest.TestCase): + def setUp(self): + self.model_config = VocoderModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + self.generator = Generator(self.model_config, self.preprocess_config) + + self.model = Discriminator(self.model_config) + + self.batch_size = 1 + self.in_length = 100 + + self.c = torch.randn( + self.batch_size, + self.preprocess_config.stft.n_mel_channels, + self.in_length, + ) + + def test_forward(self): + # Test the forward pass of the Discriminator class + x = self.generator(self.c) + + output = self.model(x) + + self.assertEqual(len(output), 2) + + # Assert MRD length + self.assertEqual(len(output[0]), 3) + + # Assert MPD length + self.assertEqual(len(output[1]), 5) + + # Test MRD output + # output_mrd = output[0] + + # fmap_mrd_dims = [ + # [ + # torch.Size([32, 1, 513]), + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 65]), + # torch.Size([32, 1, 65]), + # torch.Size([1, 1, 65]), + # ], + # [ + # torch.Size([32, 1, 1025]), + # torch.Size([32, 1, 513]), + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 129]), + # torch.Size([1, 1, 129]), + # ], + # [ + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 65]), + # torch.Size([32, 1, 33]), + # torch.Size([32, 1, 33]), + # torch.Size([1, 1, 33]), + # ], + # ] + + # for key in range(len(output[0])): + # fmap = output_mrd[key][0] + # x = output_mrd[key][1] + + # fmap_dims = fmap_mrd_dims[key] + + # # Assert the shape of the feature maps + # for i, fmap_ in enumerate(fmap): + # # Assert the feature map shape explicitly + # self.assertEqual(fmap_.shape, fmap_dims[i]) + + # # Test MPD output + # output_mpd = output[1] + + # fmap_mpd_dims = [ + # [ + # torch.Size([1, 64, 4267, 2]), + # torch.Size([1, 128, 1423, 2]), + # torch.Size([1, 256, 475, 2]), + # torch.Size([1, 512, 159, 2]), + # torch.Size([1, 1024, 159, 2]), + # torch.Size([1, 1, 159, 2]), + # ], + # [ + # torch.Size([1, 64, 2845, 3]), + # torch.Size([1, 128, 949, 3]), + # torch.Size([1, 256, 317, 3]), + # torch.Size([1, 512, 106, 3]), + # torch.Size([1, 1024, 106, 3]), + # torch.Size([1, 1, 106, 3]), + # ], + # [ + # torch.Size([1, 64, 1707, 5]), + # torch.Size([1, 128, 569, 5]), + # torch.Size([1, 256, 190, 5]), + # torch.Size([1, 512, 64, 5]), + # torch.Size([1, 1024, 64, 5]), + # torch.Size([1, 1, 64, 5]), + # ], + # [ + # torch.Size([1, 64, 1220, 7]), + # torch.Size([1, 128, 407, 7]), + # torch.Size([1, 256, 136, 7]), + # torch.Size([1, 512, 46, 7]), + # torch.Size([1, 1024, 46, 7]), + # torch.Size([1, 1, 46, 7]), + # ], + # [ + # torch.Size([1, 64, 776, 11]), + # torch.Size([1, 128, 259, 11]), + # torch.Size([1, 256, 87, 11]), + # torch.Size([1, 512, 29, 11]), + # torch.Size([1, 1024, 29, 11]), + # torch.Size([1, 1, 29, 11]), + # ], + # ] + + # for key in range(len(output[1])): + # fmap = output_mpd[key][0] + # x = output_mpd[key][1] + + # fmap_dims = fmap_mpd_dims[key] + + # # Assert the shape of the feature maps + # for i, fmap in enumerate(fmap): + # # Assert the feature map shape explicitly + # self.assertEqual(fmap.shape, fmap_dims[i]) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_discriminator_p.py b/models/vocoder/univnet/tests/test_discriminator_p.py new file mode 100644 index 0000000000000000000000000000000000000000..34d28839a4d1e90cfc72c5a0f93ca768b04c3e55 --- /dev/null +++ b/models/vocoder/univnet/tests/test_discriminator_p.py @@ -0,0 +1,120 @@ +import math +import unittest + +import torch + +from models.config import VocoderModelConfig +from models.vocoder.univnet import DiscriminatorP + + +class TestDiscriminatorP(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.channels = 1 + self.time_steps = 100 + self.period = 10 + self.model_config = VocoderModelConfig() + + self.x = torch.randn(self.batch_size, self.channels, self.time_steps) + self.model = DiscriminatorP(self.period, self.model_config) + + def test_forward(self): + fmap, output = self.model(self.x) + + self.assertEqual(len(fmap), len(self.model.convs) + 1) + + # Assert the shape of the feature maps explicitly + fmap_dims = [ + torch.Size([self.batch_size, 64, 4, self.period]), + torch.Size([self.batch_size, 128, 2, self.period]), + torch.Size([self.batch_size, 256, 1, self.period]), + torch.Size([self.batch_size, 512, 1, self.period]), + torch.Size([self.batch_size, 1024, 1, self.period]), + torch.Size([self.batch_size, 1, 1, self.period]), + ] + + for i in range(len(fmap)): + self.assertEqual(fmap[i].shape, fmap_dims[i]) + + # Assert the shape of the feature maps + dim_2nd = 4 + for i in range(len(self.model_config.mpd.periods)): + self.assertEqual(fmap[i].shape[0], self.batch_size) + self.assertEqual(fmap[i].shape[1], 2 ** (i + 6)) + self.assertEqual(fmap[i].shape[2], dim_2nd) + + dim_2nd = math.ceil(dim_2nd / self.model_config.mpd.stride) + + self.assertEqual(fmap[i].shape[3], self.period) + + self.assertEqual(output.shape, (self.batch_size, self.period)) + + def test_forward_with_padding(self): + fmap, output = self.model(self.x) + + self.assertEqual(len(fmap), len(self.model.convs) + 1) + + # Assert the shape of the feature maps explicitly + fmap_dims = [ + torch.Size([self.batch_size, 64, 4, self.period]), + torch.Size([self.batch_size, 128, 2, self.period]), + torch.Size([self.batch_size, 256, 1, self.period]), + torch.Size([self.batch_size, 512, 1, self.period]), + torch.Size([self.batch_size, 1024, 1, self.period]), + torch.Size([self.batch_size, 1, 1, self.period]), + ] + + for i in range(len(fmap)): + self.assertEqual(fmap[i].shape, fmap_dims[i]) + + # Assert the shape of the feature maps + dim_2nd = 4 + for i in range(len(self.model_config.mpd.periods)): + self.assertEqual(fmap[i].shape[0], self.batch_size) + self.assertEqual(fmap[i].shape[1], 2 ** (i + 6)) + self.assertEqual(fmap[i].shape[2], dim_2nd) + + dim_2nd = math.ceil(dim_2nd / self.model_config.mpd.stride) + + self.assertEqual(fmap[i].shape[3], self.period) + + self.assertEqual(output.shape, (self.batch_size, self.period)) + + def test_forward_with_different_period(self): + model = DiscriminatorP(self.period, self.model_config) + x = torch.randn(self.batch_size, self.channels, self.time_steps - 1) + + model.period = 5 + fmap, output = model(x) + + self.assertEqual(len(fmap), len(model.convs) + 1) + + # Assert the shape of the feature maps explicitly + fmap_dims = [ + torch.Size([self.batch_size, 64, 7, model.period]), + torch.Size([self.batch_size, 128, 3, model.period]), + torch.Size([self.batch_size, 256, 1, model.period]), + torch.Size([self.batch_size, 512, 1, model.period]), + torch.Size([self.batch_size, 1024, 1, model.period]), + torch.Size([self.batch_size, 1, 1, model.period]), + ] + + for i in range(len(fmap)): + self.assertEqual(fmap[i].shape, fmap_dims[i]) + + # Assert the shape of the feature maps + dim_2nd = 7 + for i in range(len(self.model_config.mpd.periods)): + self.assertEqual(fmap[i].shape[0], self.batch_size) + self.assertEqual(fmap[i].shape[1], 2 ** (i + 6)) + self.assertEqual(fmap[i].shape[2], dim_2nd) + + dim_2nd = math.ceil(dim_2nd / self.model_config.mpd.stride) + + self.assertEqual(fmap[i].shape[3], model.period) + + self.assertEqual(output.shape, (self.batch_size, model.period)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_discriminator_r.py b/models/vocoder/univnet/tests/test_discriminator_r.py new file mode 100644 index 0000000000000000000000000000000000000000..f29c06584ca943b722f5eb085b2af71417d0f1db --- /dev/null +++ b/models/vocoder/univnet/tests/test_discriminator_r.py @@ -0,0 +1,62 @@ +import unittest + +import torch + +from models.config import VocoderModelConfig +from models.vocoder.univnet.discriminator_r import DiscriminatorR + + +class TestDiscriminatorR(unittest.TestCase): + def setUp(self): + self.resolution = (1024, 256, 1024) + self.model_config = VocoderModelConfig() + self.model = DiscriminatorR(self.resolution, self.model_config) + + def test_forward(self): + x = torch.randn(1, 1024) + + # Test the forward pass of the DiscriminatorR class + fmap, output = self.model(x) + + self.assertEqual(len(fmap), 6) + + # Assert the shape of the feature maps explicitly + # fmap_dims = [ + # torch.Size([32, 1, 513]), + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 65]), + # torch.Size([32, 1, 65]), + # torch.Size([1, 1, 65]), + # ] + + # for i in range(len(fmap)): + # self.assertEqual(fmap[i].shape, fmap_dims[i]) + + # first_dim, second_dim = 32, 1 + + # init_p = 9 + + # def dim_3rd(p: int = init_p): + # return max(2**p + 1, 2**6 + 1) + + # # Assert the shape of the feature maps + # for i, fmap_ in enumerate(fmap[:-1]): + # self.assertEqual( + # fmap_.shape, torch.Size([first_dim, second_dim, dim_3rd(init_p - i)]), + # ) + + # self.assertEqual(fmap[-1].shape, torch.Size([second_dim, second_dim, 65])) + + self.assertEqual(output.shape, (1, 513)) + + def test_spectrogram(self): + x = torch.randn(4, 1, 16384) + # Test the spectrogram function of the DiscriminatorR class + mag = self.model.spectrogram(x) + + self.assertEqual(mag.shape, (4, 513, 64)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_generator.py b/models/vocoder/univnet/tests/test_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..2917a10e2eb4d6793a2b4ba544d204aaaad23fa8 --- /dev/null +++ b/models/vocoder/univnet/tests/test_generator.py @@ -0,0 +1,73 @@ +import unittest + +import torch +from torch import nn + +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.config import VocoderModelConfig +from models.vocoder.univnet.generator import Generator + + +class TestUnivNet(unittest.TestCase): + def setUp(self): + self.batch_size = 3 + self.in_length = 100 + + self.model_config = VocoderModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + self.generator = Generator(self.model_config, self.preprocess_config) + + self.c = torch.randn( + self.batch_size, + self.preprocess_config.stft.n_mel_channels, + self.in_length, + ) + + def test_forward(self): + output = self.generator(self.c) + + # Assert the shape + expected_shape = (self.batch_size, 1, self.in_length * 256) + self.assertEqual(output.shape, expected_shape) + + def test_generator_inference_output_shape(self): + mel_lens = torch.tensor([self.in_length] * self.batch_size) + + output = self.generator.infer(self.c, mel_lens) + + # Assert the shape + expected_shape = ( + self.batch_size, + 1, + self.in_length * self.preprocess_config.stft.hop_length, + ) + self.assertEqual(output.shape, expected_shape) + + def test_eval(self): + generator = Generator( + self.model_config, + self.preprocess_config, + ) + + generator.eval(inference=True) + for module in generator.modules(): + if isinstance(module, nn.Conv1d): + self.assertFalse(hasattr(module, "weight_g")) + self.assertFalse(hasattr(module, "weight_v")) + + def test_remove_weight_norm(self): + generator = Generator( + self.model_config, + self.preprocess_config, + ) + + generator.remove_weight_norm() + for module in generator.modules(): + if isinstance(module, nn.Conv1d): + self.assertFalse(hasattr(module, "weight_g")) + self.assertFalse(hasattr(module, "weight_v")) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_kernel_predictor.py b/models/vocoder/univnet/tests/test_kernel_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..186a7e35475bca5fe0ddc5fa0f58705fc8c1288a --- /dev/null +++ b/models/vocoder/univnet/tests/test_kernel_predictor.py @@ -0,0 +1,61 @@ +import unittest + +import torch + +from models.vocoder.univnet.kernel_predictor import KernelPredictor + + +class TestKernelPredictor(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.cond_channels = 4 + self.conv_in_channels = 3 + self.conv_out_channels = 5 + self.conv_layers = 2 + self.conv_kernel_size = 3 + self.kpnet_hidden_channels = 64 + self.kpnet_conv_size = 3 + self.kpnet_dropout = 0.0 + self.lReLU_slope = 0.1 + + self.model = KernelPredictor( + self.cond_channels, + self.conv_in_channels, + self.conv_out_channels, + self.conv_layers, + self.conv_kernel_size, + self.kpnet_hidden_channels, + self.kpnet_conv_size, + self.kpnet_dropout, + self.lReLU_slope, + ) + + def test_forward(self): + c = torch.randn(self.batch_size, self.cond_channels, 10) + kernels, bias = self.model(c) + + self.assertIsInstance(kernels, torch.Tensor) + self.assertEqual( + kernels.shape, + ( + self.batch_size, + self.conv_layers, + self.conv_in_channels, + self.conv_out_channels, + self.conv_kernel_size, + 10, + ), + ) + + self.assertIsInstance(bias, torch.Tensor) + self.assertEqual( + bias.shape, (self.batch_size, self.conv_layers, self.conv_out_channels, 10), + ) + + def test_remove_weight_norm(self): + self.model.remove_weight_norm() + + for module in self.model.modules(): + if hasattr(module, "weight_g"): + self.assertIsNone(module.weight_g) + self.assertIsNone(module.weight_v) diff --git a/models/vocoder/univnet/tests/test_lvc_block.py b/models/vocoder/univnet/tests/test_lvc_block.py new file mode 100644 index 0000000000000000000000000000000000000000..78aa4beeb59fbcd5815078615d79497d71ef3e40 --- /dev/null +++ b/models/vocoder/univnet/tests/test_lvc_block.py @@ -0,0 +1,89 @@ +import unittest + +import torch +from torch import nn + +from models.vocoder.univnet.lvc_block import LVCBlock + + +class TestLVCBlock(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.in_channels = 3 + self.cond_channels = 4 + self.stride = 2 + + self.in_length = 65536 + self.kernel_length = 10 + self.cond_length = 256 + self.dilations = [1, 3, 9, 27] + self.lReLU_slope = 0.2 + self.conv_kernel_size = 3 + self.cond_hop_length = 256 + self.kpnet_hidden_channels = 64 + self.kpnet_conv_size = 3 + self.kpnet_dropout = 0.0 + + self.x = torch.randn(self.batch_size, self.in_channels, self.in_length) + + self.kernel = torch.randn( + self.batch_size, + self.cond_channels, + self.cond_length, + ) + + self.lvc_block = LVCBlock( + in_channels=self.in_channels, + cond_channels=self.cond_channels, + stride=self.stride, + dilations=self.dilations, + lReLU_slope=self.lReLU_slope, + conv_kernel_size=self.conv_kernel_size, + cond_hop_length=self.cond_hop_length, + kpnet_hidden_channels=self.kpnet_hidden_channels, + kpnet_conv_size=self.kpnet_conv_size, + kpnet_dropout=self.kpnet_dropout, + ) + + def test_remove_weight_norm(self): + self.lvc_block.remove_weight_norm() + + for _, module in self.lvc_block.named_modules(): + if isinstance(module, (nn.Conv1d, nn.ConvTranspose1d)): + self.assertFalse(hasattr(module, "weight_g")) + self.assertFalse(hasattr(module, "weight_v")) + + def test_location_variable_convolution(self): + kernel = torch.randn( + self.batch_size, + self.in_channels, + 2 * self.in_channels, + self.conv_kernel_size, + self.cond_length, + ) + bias = torch.randn(self.batch_size, 2 * self.in_channels, self.cond_length) + + output = self.lvc_block.location_variable_convolution( + x=self.x, + kernel=kernel, + bias=bias, + dilation=1, + hop_size=self.cond_hop_length, + ) + + self.assertEqual( + output.shape, (self.batch_size, 2 * self.in_channels, self.in_length), + ) + + def test_forward(self): + x = torch.randn( + self.batch_size, + self.in_channels, + self.in_length // self.stride, + ) + + output = self.lvc_block(x, self.kernel) + + self.assertEqual( + output.shape, (self.batch_size, self.in_channels, self.in_length), + ) diff --git a/models/vocoder/univnet/tests/test_multi_period_discriminator.py b/models/vocoder/univnet/tests/test_multi_period_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..2643503d1152c81781050d34e61b75fed9001853 --- /dev/null +++ b/models/vocoder/univnet/tests/test_multi_period_discriminator.py @@ -0,0 +1,102 @@ +import math +import unittest + +import torch + +from models.config import VocoderModelConfig +from models.vocoder.univnet import MultiPeriodDiscriminator + + +class TestMultiPeriodDiscriminator(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.channels = 1 + self.time_steps = 100 + self.model_config = VocoderModelConfig() + + self.model = MultiPeriodDiscriminator(self.model_config) + self.x = torch.randn(self.batch_size, self.channels, self.time_steps) + + def test_forward(self): + output = self.model(self.x) + + self.assertEqual(len(output), len(self.model_config.mpd.periods)) + + fmaps_dims = [ + [ + torch.Size([self.batch_size, 64, 17, self.model_config.mpd.periods[0]]), + torch.Size([self.batch_size, 128, 6, self.model_config.mpd.periods[0]]), + torch.Size([self.batch_size, 256, 2, self.model_config.mpd.periods[0]]), + torch.Size([self.batch_size, 512, 1, self.model_config.mpd.periods[0]]), + torch.Size( + [self.batch_size, 1024, 1, self.model_config.mpd.periods[0]], + ), + ], + [ + torch.Size([self.batch_size, 64, 12, self.model_config.mpd.periods[1]]), + torch.Size([self.batch_size, 128, 4, self.model_config.mpd.periods[1]]), + torch.Size([self.batch_size, 256, 2, self.model_config.mpd.periods[1]]), + torch.Size([self.batch_size, 512, 1, self.model_config.mpd.periods[1]]), + torch.Size( + [self.batch_size, 1024, 1, self.model_config.mpd.periods[1]], + ), + ], + [ + torch.Size([self.batch_size, 64, 7, self.model_config.mpd.periods[2]]), + torch.Size([self.batch_size, 128, 3, self.model_config.mpd.periods[2]]), + torch.Size([self.batch_size, 256, 1, self.model_config.mpd.periods[2]]), + torch.Size([self.batch_size, 512, 1, self.model_config.mpd.periods[2]]), + torch.Size( + [self.batch_size, 1024, 1, self.model_config.mpd.periods[2]], + ), + ], + [ + torch.Size([self.batch_size, 64, 5, self.model_config.mpd.periods[3]]), + torch.Size([self.batch_size, 128, 2, self.model_config.mpd.periods[3]]), + torch.Size([self.batch_size, 256, 1, self.model_config.mpd.periods[3]]), + torch.Size([self.batch_size, 512, 1, self.model_config.mpd.periods[3]]), + torch.Size( + [self.batch_size, 1024, 1, self.model_config.mpd.periods[3]], + ), + ], + [ + torch.Size([self.batch_size, 64, 4, self.model_config.mpd.periods[4]]), + torch.Size([self.batch_size, 128, 2, self.model_config.mpd.periods[4]]), + torch.Size([self.batch_size, 256, 1, self.model_config.mpd.periods[4]]), + torch.Size([self.batch_size, 512, 1, self.model_config.mpd.periods[4]]), + torch.Size( + [self.batch_size, 1024, 1, self.model_config.mpd.periods[4]], + ), + ], + ] + + init_2nd_dims = [17, 12, 7, 5, 4] + + for mpd_k in range(len(self.model_config.mpd.periods)): + fmap = output[mpd_k][0] + x = output[mpd_k][1] + + self.assertEqual(len(x), self.batch_size) + + # Assert the shape of the feature maps + dim_2nd = init_2nd_dims[mpd_k] + period = self.model_config.mpd.periods[mpd_k] + + dims_expl = fmaps_dims[mpd_k] + + for i in range(len(self.model_config.mpd.periods)): + # Assert the shape of the feature maps explicitly + self.assertEqual(fmap[i].shape, dims_expl[i]) + + # Assert the shape of the feature maps + self.assertEqual( + fmap[i].shape, + torch.Size([self.batch_size, 2 ** (i + 6), dim_2nd, period]), + ) + dim_2nd = math.ceil(dim_2nd / self.model_config.mpd.stride) + + self.assertEqual(len(output), len(self.model_config.mpd.periods)) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_multi_resolution_discriminator.py b/models/vocoder/univnet/tests/test_multi_resolution_discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..0f84e76ee60dffc88eed152c6efe4c4887cc508c --- /dev/null +++ b/models/vocoder/univnet/tests/test_multi_resolution_discriminator.py @@ -0,0 +1,76 @@ +import unittest + +import torch + +from models.config import VocoderModelConfig +from models.vocoder.univnet import MultiResolutionDiscriminator + + +class TestMultiResolutionDiscriminator(unittest.TestCase): + def setUp(self): + self.resolution = [(1024, 256, 1024), (2048, 512, 2048)] + self.model_config = VocoderModelConfig() + self.model = MultiResolutionDiscriminator(self.model_config) + + self.x = torch.randn(1, 1024) + + def test_forward(self): + # Test the forward pass of the MultiResolutionDiscriminator class + output = self.model(self.x) + self.assertEqual(len(output), 3) + + # fmap_dims = [ + # [ + # torch.Size([32, 1, 513]), + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 65]), + # torch.Size([32, 1, 65]), + # torch.Size([1, 65]), + # ], + # [ + # torch.Size([32, 1, 1025]), + # torch.Size([32, 1, 513]), + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 129]), + # torch.Size([1, 129]), + # ], + # [ + # torch.Size([32, 1, 257]), + # torch.Size([32, 1, 129]), + # torch.Size([32, 1, 65]), + # torch.Size([32, 1, 33]), + # torch.Size([32, 1, 33]), + # torch.Size([1, 33]), + # ], + # ] + + # init_powers_max_min = [(9, 6), (10, 7), (8, 5)] + + # for key in range(len(output)): + # fmap = output[key][0] + + # first_dim, second_dim = 32, 1 + + # p_max, p_min = init_powers_max_min[key] + + # def dim_3rd(p: int): + # return max(2**p + 1, 2**p_min + 1) + + # fmap_dim = fmap_dims[key] + + # # Assert the shape of the feature maps + # for i, fmap_ in enumerate(fmap[:-1]): + # # Assert the feature map shape explicitly + # self.assertEqual(fmap_.shape, fmap_dim[i]) + + # self.assertEqual( + # fmap_.shape, torch.Size([first_dim, second_dim, dim_3rd(p_max - i)]), + # ) + + # self.assertEqual(fmap[-1].shape, torch.Size([second_dim, second_dim, 2**p_min + 1])) + + +if __name__ == "__main__": + unittest.main() diff --git a/models/vocoder/univnet/tests/test_traced_generator.py b/models/vocoder/univnet/tests/test_traced_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..ecdbe543f10cc75f7b99ad544fe4504dd776ddc1 --- /dev/null +++ b/models/vocoder/univnet/tests/test_traced_generator.py @@ -0,0 +1,60 @@ +import unittest + +import torch + +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.config import VocoderModelConfig +from models.helpers.tools import get_mask_from_lengths +from models.vocoder.univnet import Generator, TracedGenerator + + +class TestTracedUnivNet(unittest.TestCase): + def setUp(self): + self.batch_size = 3 + self.in_length = 100 + self.mel_channels = 80 + + self.model_config = VocoderModelConfig() + self.preprocess_config = PreprocessingConfig("english_only") + + self.generator = Generator(self.model_config, self.preprocess_config) + + self.example_inputs = ( + torch.randn( + self.batch_size, + self.preprocess_config.stft.n_mel_channels, + self.in_length, + ), + ) + + self.traced_generator = TracedGenerator( + self.generator + ) # , self.example_inputs) + + self.c = torch.randn( + self.batch_size, + self.preprocess_config.stft.n_mel_channels, + self.in_length, + ) + + self.mel_lens = torch.tensor([self.in_length] * self.batch_size) + + def test_forward(self): + output = self.traced_generator(self.c, self.mel_lens) + + # Assert the shape + expected_shape = (self.batch_size, 1, self.in_length * 256) + self.assertEqual(output.shape, expected_shape) + + def test_forward_with_masked_c(self): + mel_lens = torch.tensor([self.in_length] * self.batch_size) + + # Mask the input mel-spectrogram tensor + mel_mask = get_mask_from_lengths(mel_lens).unsqueeze(1) + c = self.c.masked_fill(mel_mask, self.traced_generator.mel_mask_value) + + output = self.traced_generator(c, mel_lens) + + # Assert the shape + expected_shape = (self.batch_size, 1, self.in_length * 256) + self.assertEqual(output.shape, expected_shape) diff --git a/models/vocoder/univnet/tests/test_univnet.py b/models/vocoder/univnet/tests/test_univnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f8508c5e51998745953653b96749c2e335c6e9d1 --- /dev/null +++ b/models/vocoder/univnet/tests/test_univnet.py @@ -0,0 +1,64 @@ +import os +import unittest + +from lightning.pytorch import Trainer +import torch + +from models.config import VocoderFinetuningConfig, VocoderPretrainingConfig +from models.vocoder.univnet import UnivNet + +# NOTE: this is needed to avoid CUDA_LAUNCH_BLOCKING error +os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + +class TestUnivNet(unittest.TestCase): + def test_optim_finetuning(self): + module = UnivNet() + + self.assertIsInstance(module.train_config, VocoderPretrainingConfig) + + optimizer_configs = module.configure_optimizers() + + for optimizer_config in optimizer_configs: + optimizer = optimizer_config["optimizer"] + lr_scheduler = optimizer_config["lr_scheduler"] + + # Test the optimizer + self.assertIsInstance(optimizer, torch.optim.AdamW) + self.assertIsInstance(lr_scheduler, torch.optim.lr_scheduler.ExponentialLR) + + def test_finetuning(self): + module = UnivNet(fine_tuning=True) + + self.assertIsInstance(module.train_config, VocoderFinetuningConfig) + + def test_train_step(self): + trainer = Trainer( + # Save checkpoints to the `default_root_dir` directory + default_root_dir="checkpoints/vocoder", + limit_train_batches=1, + max_epochs=1, + accelerator="cpu", + ) + + # Load the pretrained weights + # NOTE: this is the path to the checkpoint in the repo + # It works only for version 0.1.0 checkpoint + # This code will be removed in the future! + # checkpoint_path = "models/checkpoints/assets/v0.1.0/vocoder_pretrained.pt" + + module = UnivNet(batch_size=1, acc_grad_steps=1) + + train_dataloader = module.train_dataloader(2, cache=False, mem_cache=False) + + result = trainer.fit(model=module, train_dataloaders=train_dataloader) + + self.assertIsNone(result) + + # def test_load_from_checkpoint(self): + # try: + # UnivNet.load_from_checkpoint( + # "./checkpoints/vocoder.ckpt", + # ) + # except Exception as e: + # self.fail(f"Loading from checkpoint raised an exception: {e}") + diff --git a/models/vocoder/univnet/traced_generator.py b/models/vocoder/univnet/traced_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..28ac623983b6f0b7951434f722299ca5bd6ad43e --- /dev/null +++ b/models/vocoder/univnet/traced_generator.py @@ -0,0 +1,52 @@ +import torch +from torch.nn import Module + +from models.helpers.tools import get_mask_from_lengths + +from .generator import Generator + + +class TracedGenerator(Module): + def __init__( + self, + generator: Generator, + # example_inputs: Tuple[Any], example_inputs (Tuple[Any]): Example inputs to use for tracing. + ): + r"""A traced version of the UnivNet class that can be used for faster inference. + + Args: + generator (UnivNet): The UnivNet instance to trace. + """ + super().__init__() + + self.mel_mask_value: float = generator.mel_mask_value + self.hop_length: int = generator.hop_length + + # TODO: https://github.com/Lightning-AI/lightning/issues/14036 + # Disable trace since model is non-deterministic + # self.generator = torch.jit.trace(generator, example_inputs, check_trace=False) + # self.generator = generator.to_torchscript( + # method="trace", example_inputs=example_inputs, check_trace=False + # ) + self.generator = generator + + def forward(self, c: torch.Tensor, mel_lens: torch.Tensor) -> torch.Tensor: + r"""Forward pass of the traced UnivNet. + + Args: + c (torch.Tensor): The input mel-spectrogram tensor. + mel_lens (torch.Tensor): The lengths of the input mel-spectrograms. + + Returns: + torch.Tensor: The generated audio tensor. + """ + mel_mask = get_mask_from_lengths(mel_lens).unsqueeze(1).to(c.device) + c = c.masked_fill(mel_mask, self.mel_mask_value) + zero = torch.full( + (c.shape[0], c.shape[1], 10), self.mel_mask_value, device=c.device, + ) + mel = torch.cat((c, zero), dim=2) + audio = self.generator(mel) + audio = audio[:, :, : -(self.hop_length * 10)] + audio_mask = get_mask_from_lengths(mel_lens * 256).unsqueeze(1) + return audio.masked_fill(audio_mask, 0.0) diff --git a/models/vocoder/univnet/train.py b/models/vocoder/univnet/train.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ed56f97a2ba1bfb5b4cb1984454311ab633b23 --- /dev/null +++ b/models/vocoder/univnet/train.py @@ -0,0 +1,81 @@ +from datetime import datetime +import logging +import sys + +from lightning.pytorch import Trainer +from lightning.pytorch.accelerators import find_usable_cuda_devices # type: ignore +from lightning.pytorch.strategies import DDPStrategy +import torch + +from models.vocoder.univnet import UnivNet + +# Get the current date and time +now = datetime.now() + +# Format the current date and time as a string +timestamp = now.strftime("%Y%m%d_%H%M%S") + +# Create a logger +logger = logging.getLogger("my_logger") + +# Set the level of the logger to ERROR +logger.setLevel(logging.ERROR) + +# Create a file handler that logs error messages to a file with the current timestamp in its name +handler = logging.FileHandler(f"logs/error_{timestamp}.log") + +# Create a formatter and add it to the handler +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) + +# Add the handler to the logger +logger.addHandler(handler) + + +print("usable_cuda_devices: ", find_usable_cuda_devices()) + +# Set the precision of the matrix multiplication to float32 to improve the performance of the training +torch.set_float32_matmul_precision("high") + +default_root_dir = "logs" + +# ckpt_acoustic="./checkpoints/epoch=301-step=124630.ckpt" + +# ckpt_vocoder="./checkpoints/vocoder.ckpt" + +try: + trainer = Trainer( + accelerator="cuda", + devices=-1, + strategy=DDPStrategy( + gradient_as_bucket_view=True, + find_unused_parameters=True, + ), + # Save checkpoints to the `default_root_dir` directory + default_root_dir=default_root_dir, + enable_checkpointing=True, + max_epochs=-1, + log_every_n_steps=10, + ) + + model = UnivNet() + + train_dataloader = model.train_dataloader( + # NOTE: Preload the cached dataset into the RAM + cache_dir="/dev/shm/", + cache=True, + mem_cache=False, + ) + + trainer.fit( + model=model, + train_dataloaders=train_dataloader, + # val_dataloaders=val_dataloader, + # Resume training states from the checkpoint file + # ckpt_path=ckpt_acoustic, + ) + +except Exception as e: + # Log the error message + logger.error(f"An error occurred: {e}") + sys.exit(1) diff --git a/models/vocoder/univnet/univnet.py b/models/vocoder/univnet/univnet.py new file mode 100644 index 0000000000000000000000000000000000000000..a64a85aa060eae3da2de0b33d0bf2a7027ff8f56 --- /dev/null +++ b/models/vocoder/univnet/univnet.py @@ -0,0 +1,327 @@ +from typing import List, Optional, Tuple + +from lightning.pytorch.core import LightningModule +import torch +from torch.optim import AdamW, Optimizer, swa_utils +from torch.optim.lr_scheduler import ExponentialLR +from torch.utils.data import DataLoader + +from models.config import ( + PreprocessingConfigUnivNet as PreprocessingConfig, +) +from models.config import ( + VocoderFinetuningConfig, + VocoderModelConfig, + VocoderPretrainingConfig, + VoicoderTrainingConfig, +) +from models.helpers.dataloaders import train_dataloader +from training.loss import UnivnetLoss + +from .discriminator import Discriminator +from .generator import Generator + + +class UnivNet(LightningModule): + r"""Univnet module. + + This module contains the `Generator` and `Discriminator` models, and handles training and optimization. + """ + + def __init__( + self, + fine_tuning: bool = False, + lang: str = "en", + acc_grad_steps: int = 10, + batch_size: int = 6, + root: str = "datasets_cache/LIBRITTS", + checkpoint_path_v1: Optional[str] = "vocoder_pretrained.pt", + ): + r"""Initializes the `VocoderModule`. + + Args: + fine_tuning (bool, optional): Whether to use fine-tuning mode or not. Defaults to False. + lang (str): Language of the dataset. + acc_grad_steps (int): Accumulated gradient steps. + batch_size (int): The batch size. + root (str, optional): The root directory for the dataset. Defaults to "datasets_cache/LIBRITTS". + checkpoint_path_v1 (str, optional): The path to the checkpoint for the model. If provided, the model weights will be loaded from this checkpoint. Defaults to None. + """ + super().__init__() + + # Switch to manual optimization + self.automatic_optimization = False + self.acc_grad_steps = acc_grad_steps + self.batch_size = batch_size + + self.lang = lang + self.root = root + + model_config = VocoderModelConfig() + preprocess_config = PreprocessingConfig("english_only") + + self.univnet = Generator( + model_config=model_config, + preprocess_config=preprocess_config, + ) + self.discriminator = Discriminator(model_config=model_config) + + # Initialize SWA + self.swa_averaged_univnet = swa_utils.AveragedModel(self.univnet) + self.swa_averaged_discriminator = swa_utils.AveragedModel(self.discriminator) + + self.loss = UnivnetLoss() + + self.train_config: VoicoderTrainingConfig = ( + VocoderFinetuningConfig() if fine_tuning else VocoderPretrainingConfig() + ) + + # NOTE: this code is used only for the v0.1.0 checkpoint. + # In the future, this code will be removed! + self.checkpoint_path_v1 = checkpoint_path_v1 + if checkpoint_path_v1 is not None: + generator, discriminator, _, _ = self.get_weights_v1(checkpoint_path_v1) + self.univnet.load_state_dict(generator, strict=False) + self.discriminator.load_state_dict(discriminator, strict=False) + + def get_weights_v1(self, checkpoint_path: str) -> Tuple[dict, dict, dict, dict]: + r"""NOTE: this method is used only for the v0.1.0 checkpoint. + Prepares the weights for the model. + + This is required for the model to be loaded from the checkpoint. + + Args: + checkpoint_path (str): The path to the checkpoint. + + Returns: + Tuple[dict, dict, dict, dict]: The weights for the generator and discriminator. + """ + ckpt_acoustic = torch.load(checkpoint_path, map_location=torch.device("cpu")) + + return ( + ckpt_acoustic["generator"], + ckpt_acoustic["discriminator"], + ckpt_acoustic["optim_g"], + ckpt_acoustic["optim_d"], + ) + + def forward(self, y_pred: torch.Tensor) -> torch.Tensor: + r"""Performs a forward pass through the UnivNet model. + + Args: + y_pred (torch.Tensor): The predicted mel spectrogram. + + Returns: + torch.Tensor: The output of the UnivNet model. + """ + mel_lens = torch.tensor( + [y_pred.shape[2]], + dtype=torch.int32, + device=y_pred.device, + ) + + wav_prediction = self.univnet.infer(y_pred, mel_lens) + + return wav_prediction[0, 0] + + def training_step(self, batch: List, batch_idx: int): + r"""Performs a training step for the model. + + Args: + batch (List): The batch of data for training. The batch should contain the mel spectrogram, its length, the audio, and the speaker ID. + batch_idx (int): Index of the batch. + + Returns: + dict: A dictionary containing the total loss for the generator and logs for tensorboard. + """ + ( + _, + _, + _, + _, + _, + mels, + _, + _, + _, + _, + _, + wavs, + _, + ) = batch + + # Access your optimizers + optimizers = self.optimizers() + schedulers = self.lr_schedulers() + opt_univnet: Optimizer = optimizers[0] # type: ignore + sch_univnet: ExponentialLR = schedulers[0] # type: ignore + + opt_discriminator: Optimizer = optimizers[1] # type: ignore + sch_discriminator: ExponentialLR = schedulers[1] # type: ignore + + audio = wavs + fake_audio = self.univnet(mels) + + res_fake, period_fake = self.discriminator(fake_audio.detach()) + res_real, period_real = self.discriminator(audio) + + ( + total_loss_gen, + total_loss_disc, + stft_loss, + score_loss, + esr_loss, + snr_loss, + ) = self.loss.forward( + audio, + fake_audio, + res_fake, + period_fake, + res_real, + period_real, + ) + + self.log( + "total_loss_gen", + total_loss_gen, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log( + "total_loss_disc", + total_loss_disc, + sync_dist=True, + batch_size=self.batch_size, + ) + self.log("stft_loss", stft_loss, sync_dist=True, batch_size=self.batch_size) + self.log("esr_loss", esr_loss, sync_dist=True, batch_size=self.batch_size) + self.log("snr_loss", snr_loss, sync_dist=True, batch_size=self.batch_size) + self.log("score_loss", score_loss, sync_dist=True, batch_size=self.batch_size) + + # Perform manual optimization + self.manual_backward(total_loss_gen / self.acc_grad_steps, retain_graph=True) + self.manual_backward(total_loss_disc / self.acc_grad_steps, retain_graph=True) + + # accumulate gradients of N batches + if (batch_idx + 1) % self.acc_grad_steps == 0: + # clip gradients + self.clip_gradients( + opt_univnet, + gradient_clip_val=0.5, + gradient_clip_algorithm="norm", + ) + self.clip_gradients( + opt_discriminator, + gradient_clip_val=0.5, + gradient_clip_algorithm="norm", + ) + + # optimizer step + opt_univnet.step() + opt_discriminator.step() + + # Scheduler step + sch_univnet.step() + sch_discriminator.step() + + # zero the gradients + opt_univnet.zero_grad() + opt_discriminator.zero_grad() + + def configure_optimizers(self): + r"""Configures the optimizers and learning rate schedulers for the `UnivNet` and `Discriminator` models. + + This method creates an `AdamW` optimizer and an `ExponentialLR` scheduler for each model. + The learning rate, betas, and decay rate for the optimizers and schedulers are taken from the training configuration. + + Returns + tuple: A tuple containing two dictionaries. Each dictionary contains the optimizer and learning rate scheduler for one of the models. + + Examples + ```python + vocoder_module = VocoderModule() + optimizers = vocoder_module.configure_optimizers() + + print(optimizers) + ( + {"optimizer": , "lr_scheduler": }, + {"optimizer": , "lr_scheduler": } + ) + ``` + """ + optim_univnet = AdamW( + self.univnet.parameters(), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_univnet = ExponentialLR( + optim_univnet, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + optim_discriminator = AdamW( + self.discriminator.parameters(), + self.train_config.learning_rate, + betas=(self.train_config.adam_b1, self.train_config.adam_b2), + ) + scheduler_discriminator = ExponentialLR( + optim_discriminator, + gamma=self.train_config.lr_decay, + last_epoch=-1, + ) + + # NOTE: this code is used only for the v0.1.0 checkpoint. + # In the future, this code will be removed! + if self.checkpoint_path_v1 is not None: + _, _, optim_g, optim_d = self.get_weights_v1(self.checkpoint_path_v1) + optim_univnet.load_state_dict(optim_g) + optim_discriminator.load_state_dict(optim_d) + + return ( + {"optimizer": optim_univnet, "lr_scheduler": scheduler_univnet}, + {"optimizer": optim_discriminator, "lr_scheduler": scheduler_discriminator}, + ) + + def on_train_epoch_end(self): + r"""Updates the averaged model after each optimizer step with SWA.""" + self.swa_averaged_univnet.update_parameters(self.univnet) + self.swa_averaged_discriminator.update_parameters(self.discriminator) + + def on_train_end(self): + # Update SWA model after training + swa_utils.update_bn(self.train_dataloader(), self.swa_averaged_univnet) + swa_utils.update_bn(self.train_dataloader(), self.swa_averaged_discriminator) + + def train_dataloader( + self, + num_workers: int = 5, + root: str = "datasets_cache/LIBRITTS", + cache: bool = True, + cache_dir: str = "datasets_cache", + mem_cache: bool = False, + url: str = "train-clean-360", + ) -> DataLoader: + r"""Returns the training dataloader, that is using the LibriTTS dataset. + + Args: + num_workers (int): The number of workers. + root (str): The root directory of the dataset. + cache (bool): Whether to cache the preprocessed data. + cache_dir (str): The directory for the cache. + mem_cache (bool): Whether to use memory cache. + url (str): The URL of the dataset. + + Returns: + DataLoader: The training and validation dataloaders. + """ + return train_dataloader( + batch_size=self.batch_size, + num_workers=num_workers, + root=root, + cache=cache, + cache_dir=cache_dir, + mem_cache=mem_cache, + url=url, + lang=self.lang, + ) diff --git a/packages.txt b/packages.txt new file mode 100644 index 0000000000000000000000000000000000000000..a210b44cb6932d46a7a0ce4c03e31c7dd7821d6f --- /dev/null +++ b/packages.txt @@ -0,0 +1,5 @@ +ffmpeg +libsndfile1 +libasound2-dev +build-essential +espeak-ng \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a79e1dc4daf50a5ef6cb74ed9028b888424ddc40 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +Cython +torch +torchaudio +torchvision +torchmetrics +lightning +gradio==4.36.0 +nemo-text-processing +unidecode +deep-phonemizer +piq +auraloss +pesq +lhotse +matplotlib==3.9.0 +ffmpeg==1.4 +ffmpy==0.3.2 +lhotse==1.22.0 +librosa==0.10.0 +nltk==3.8.1 +phonemizer==3.2.1 +pydub==0.25.1 +pynini==2.1.5 +pystoi==0.4.1 +tokenizers==0.19.1 +voicefixer==0.1.3 +git+https://github.com/detly/gammatone.git \ No newline at end of file diff --git a/training/__init__.py b/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/training/datasets/__init__.py b/training/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6da14fa0cd00538223d4aeda4840286dd1c68adb --- /dev/null +++ b/training/datasets/__init__.py @@ -0,0 +1,6 @@ +from .hifi_gan_dataset import HifiGanDataset +from .hifi_libri_dataset import HifiLibriDataset +from .libritts_dataset_acoustic import LibriTTSDatasetAcoustic +from .libritts_dataset_vocoder import LibriTTSDatasetVocoder +from .libritts_mm_dataset_acoustic import LibriTTSMMDatasetAcoustic +from .preprocessed_dataset import PreprocessedDataset diff --git a/training/datasets/dataset_preparation.ipynb b/training/datasets/dataset_preparation.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ea1ca88c393279f4c25c91668aefed273075dba7 --- /dev/null +++ b/training/datasets/dataset_preparation.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "9d8dbc75-dacc-457f-890c-c3d70f1e24ff", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "from lhotse import CutSet\n", + "from lhotse.cut import Cut, MonoCut\n", + "from lhotse.recipes import (\n", + " download_voxceleb1,\n", + " download_voxceleb2,\n", + " hifitts,\n", + " libritts,\n", + " prepare_voxceleb,\n", + ")\n", + "from lhotse.dataset import (\n", + " SimpleCutSampler,\n", + " UnsupervisedDataset\n", + ")\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "181a454e-1e74-40fd-8d5b-765b65bf6111", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Dataset root path\n", + "root_dir = Path(\"datasets_cache\")\n", + "\n", + "# datasets paths\n", + "hifitts_path = root_dir / \"hifitts\"\n", + "libritts_path = root_dir / \"libritts\"\n", + "librittsr_path = root_dir / \"librittsr\"\n", + "\n", + "num_jobs = os.cpu_count() - 3" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7e6c3a68-b4e9-46ea-89b2-4aeed61f4a64", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading LibriTTS parts: 0%| | 0/1 [00:00 Path: + r"""Calculate the path to the cache subdirectory. + + Args: + idx (int): The index of the cache subdirectory. + + Returns: + Path: The path to the cache subdirectory. + """ + return self.cache_dir / str(((idx // 1000) + 1) * 1000) + + def get_cache_file_path(self, idx: int) -> Path: + r"""Calculate the path to the cache file. + + Args: + idx (int): The index of the cache file. + + Returns: + Path: The path to the cache file. + """ + return self.get_cache_subdir_path(idx) / f"{idx}.pt" + + def __getitem__(self, idx: int) -> Tuple[str, Tensor, Tensor]: + r"""Get an item from the dataset. + + If caching is enabled and the item is in the cache, the cached item is returned. + Otherwise, the item is loaded from the dataset, preprocessed, and returned. + + Args: + idx (int): The index of the item in the dataset. + + Returns: + Tuple[str, Tensor, Tensor]: The ID of the item, the audio waveform, and the mel spectrogram. + """ + cache_file = self.get_cache_file_path(idx) + + if self.cache and cache_file.exists(): + cached_data: Tuple[str, Tensor, Tensor] = torch.load(cache_file) + return cached_data + + item = self.dataset[idx] + frames_per_seg = math.ceil(self.segment_size / self.hop_size) + + audio = item.wav + mel = item.mel.unsqueeze(0) + + if audio.size(1) >= self.segment_size: + mel_start = random.randint(0, mel.size(2) - frames_per_seg - 1) # noqa: S311 + mel = mel[:, :, mel_start : mel_start + frames_per_seg] + audio = audio[ + :, + mel_start * self.hop_size : (mel_start + frames_per_seg) + * self.hop_size, + ] + else: + mel = F.pad( + mel, + (0, frames_per_seg - mel.size(2)), + "constant", + ) + audio = F.pad( + audio, + (0, self.segment_size - audio.size(1)), + "constant", + ) + + result = (item.id, audio, mel.squeeze(0)) + + if self.cache: + # Create the cache subdirectory if it doesn't exist + Path.mkdir( + self.get_cache_subdir_path(idx), + parents=True, + exist_ok=True, + ) + # Save the preprocessed data to the cache + torch.save(result, cache_file) + + return result + + def __iter__(self): + r"""Method makes the class iterable. It iterates over the `_walker` attribute + and for each item, it gets the corresponding item from the dataset using the + `__getitem__` method. + + Yields: + The item from the dataset corresponding to the current item in `_walker`. + """ + for item in range(self.__len__()): + yield self.__getitem__(item) + + +def train_dataloader( + lang: str = "en", + root: str = "datasets_cache", + sampling_rate: int = 44100, + hifitts_path: str = "hifitts", + hifi_cutset_file_name: str = "hifi.json.gz", + libritts_path: str = "librittsr", + libritts_cutset_file_name: str = "libri.json.gz", + libritts_subsets: List[str] | str = "all", + cache: bool = False, + cache_dir: str = "/dev/shm", + num_jobs: int = NUM_JOBS, + num_workers: int = 0, + shuffle: bool = False, + batch_size: int = 5, + pin_memory: bool = True, + drop_last: bool = True, +) -> DataLoader: + r"""Create a DataLoader for the training data. + + Args: + lang (str, optional): The language of the dataset. Defaults to "en". + root (str, optional): The root directory of the dataset. Defaults to "datasets_cache". + sampling_rate (int, optional): The sampling rate of the audio. Defaults to 44100. + hifitts_path (str, optional): The path to the HiFiTTS dataset. Defaults to "hifitts". + hifi_cutset_file_name (str, optional): The file name of the HiFiTTS cutset. Defaults to "hifi.json.gz". + libritts_path (str, optional): The path to the LibriTTS dataset. Defaults to "librittsr". + libritts_cutset_file_name (str, optional): The file name of the LibriTTS cutset. Defaults to "libri.json.gz". + libritts_subsets (Union[List[str], str], optional): The subsets of the LibriTTS dataset to use. Defaults to "all". + cache (bool, optional): Whether to cache the dataset. Defaults to False. + cache_dir (str, optional): The directory to cache the dataset in. Defaults to "/dev/shm". + num_jobs (int, optional): The number of jobs to use for preparing the dataset. Defaults to NUM_JOBS. + num_workers (int, optional): The number of worker processes to use for loading the data. Defaults to 0. + shuffle (bool, optional): Whether to shuffle the data. Defaults to False. + batch_size (int, optional): The batch size. Defaults to 5. + pin_memory (bool, optional): Whether to pin memory. Defaults to True. + drop_last (bool, optional): Whether to drop the last incomplete batch. Defaults to True. + num_gpus (int, optional): The number of GPUs to use. Defaults to 1. + + Returns: + DataLoader: A DataLoader for the training data. + """ + trainset = HifiGanDataset( + lang=lang, + root=root, + sampling_rate=sampling_rate, + hifitts_path=hifitts_path, + hifi_cutset_file_name=hifi_cutset_file_name, + libritts_path=libritts_path, + libritts_cutset_file_name=libritts_cutset_file_name, + libritts_subsets=libritts_subsets, + cache=cache, + cache_dir=cache_dir, + num_jobs=num_jobs, + ) + + train_loader = DataLoader( + trainset, + num_workers=num_workers, + shuffle=shuffle, + batch_size=batch_size, + pin_memory=pin_memory, + drop_last=drop_last, + ) + + return train_loader diff --git a/training/datasets/hifi_libri_dataset.py b/training/datasets/hifi_libri_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..121cca92f6e7a9499eed333ec18f39c73a678b85 --- /dev/null +++ b/training/datasets/hifi_libri_dataset.py @@ -0,0 +1,623 @@ +from dataclasses import asdict, dataclass +import os +from pathlib import Path +import tempfile +from typing import Dict, List, Literal, Optional, Tuple + +os.environ["TQDM_DISABLE"] = "1" +import tqdm + + +def nop(it, *a, **k): + return it + + +tqdm.tqdm = nop + +from lhotse import CutSet, RecordingSet, SupervisionSet +from lhotse.cut import MonoCut +from lhotse.recipes import hifitts, libritts +import numpy as np +import soundfile as sf +import torch +from torch import Tensor +from torch.utils.data import DataLoader, Dataset +from voicefixer import VoiceFixer + +from models.config import PreprocessingConfigHifiGAN as PreprocessingConfig +from models.config import get_lang_map, lang2id +from training.preprocess import PreprocessLibriTTS +from training.tools import pad_1D, pad_2D, pad_3D + +NUM_JOBS = (os.cpu_count() or 2) - 1 + + +# The selected speakers from the HiFiTTS dataset +speakers_hifi_ids = [ + "Cori Samuel", # 92, + "Tony Oliva", # 6671, + "John Van Stan", # 9017, + "Helen Taylor", # 9136, + # "Phil Benson", # 6097, + # "Mike Pelton", # 6670, + # "Maria Kasper", # 8051, + # "Sylviamb", # 11614, + # "Celine Major", # 11697, + # "LikeManyWaters", # 12787, +] + +# The selected speakers from the LibriTTS dataset +speakers_libri_ids = list( + map( + str, + [ + # train-clean-100 + 40, + 1088, + # train-clean-360 + 3307, + 5935, + # train-other-500 + 215, + 6594, + 3867, + 5733, + 5181, + ], + ), +) + +# Map the speaker ids to string and list of selected speaker ids to set +selected_speakers_ids = { + v: k + for k, v in enumerate( + speakers_hifi_ids + speakers_libri_ids, + ) +} + + +def prep_2_cutset(prep: Dict[str, Dict[str, RecordingSet | SupervisionSet]]) -> CutSet: + r"""Prepare the dataset for the model. This function is used to convert the prepared dataset to a CutSet. + + Args: + prep (Dict[str, Dict[str, RecordingSet | SupervisionSet]]): The prepared dataset. + + Returns: + CutSet: The dataset prepared for the model. + """ + recordings_hifi = RecordingSet() + supervisions_hifi = SupervisionSet() + + for hifi_row in prep.values(): + record = hifi_row["recordings"] + supervision = hifi_row["supervisions"] + + # Separate the recordings and supervisions + if isinstance(record, RecordingSet): + recordings_hifi += record + + if isinstance(supervision, SupervisionSet): + supervisions_hifi += supervision + + # Add the recordings and supervisions to the CutSet + return CutSet.from_manifests( + recordings=recordings_hifi, + supervisions=supervisions_hifi, + ) + + +DATASET_TYPES = Literal["hifitts", "libritts"] + + +@dataclass +class HifiLibriItem: + """Dataset row for the HiFiTTS and LibriTTS datasets combined in this code. + + Args: + id (str): The ID of the item. + wav (Tensor): The waveform of the audio. + mel (Tensor): The mel spectrogram. + pitch (Tensor): The pitch. + text (Tensor): The text. + attn_prior (Tensor): The attention prior. + energy (Tensor): The energy. + raw_text (str): The raw text. + normalized_text (str): The normalized text. + speaker (int): The speaker ID. + pitch_is_normalized (bool): Whether the pitch is normalized. + lang (int): The language ID. + dataset_type (DATASET_TYPES): The type of dataset. + """ + + id: str + wav: Tensor + mel: Tensor + pitch: Tensor + text: Tensor + attn_prior: Tensor + energy: Tensor + raw_text: str + normalized_text: str + speaker: int + pitch_is_normalized: bool + lang: int + dataset_type: DATASET_TYPES + + +class HifiLibriDataset(Dataset): + r"""A PyTorch dataset for loading delightful TTS data.""" + + def __init__( + self, + lang: str = "en", + root: str = "datasets_cache", + sampling_rate: int = 44100, + hifitts_path: str = "hifitts", + hifi_cutset_file_name: str = "hifi.json.gz", + libritts_path: str = "librittsr", + libritts_cutset_file_name: str = "libri.json.gz", + libritts_subsets: List[str] | str = "all", + cache: bool = False, + cache_dir: str = "/dev/shm", + num_jobs: int = NUM_JOBS, + min_seconds: Optional[float] = None, + max_seconds: Optional[float] = None, + include_libri: bool = True, + libri_speakers: List[str] = speakers_libri_ids, + hifi_speakers: List[str] = speakers_hifi_ids, + ): + r"""Initializes the dataset. + + Args: + lang (str, optional): The language of the dataset. Defaults to "en". + root (str, optional): The root directory of the dataset. Defaults to "datasets_cache". + sampling_rate (int, optional): The sampling rate of the audio. Defaults to 44100. + hifitts_path (str, optional): The path to the HiFiTTS dataset. Defaults to "hifitts". + hifi_cutset_file_name (str, optional): The file name of the HiFiTTS cutset. Defaults to "hifi.json.gz". + libritts_path (str, optional): The path to the LibriTTS dataset. Defaults to "librittsr". + libritts_cutset_file_name (str, optional): The file name of the LibriTTS cutset. Defaults to "libri.json.gz". + libritts_subsets (Union[List[str], str], optional): The subsets of the LibriTTS dataset to use. Defaults to "all". + cache (bool, optional): Whether to cache the dataset. Defaults to False. + cache_dir (str, optional): The directory to cache the dataset in. Defaults to "/dev/shm". + num_jobs (int, optional): The number of jobs to use for preparing the dataset. Defaults to NUM_JOBS. + min_seconds (Optional[float], optional): The minimum duration of the audio. Defaults from the preprocess config. + max_seconds (Optional[float], optional): The maximum duration of the audio. Defaults from the preprocess config. + include_libri (bool, optional): Whether to include the LibriTTS dataset. Defaults to True. + libri_speakers (List[str], optional): The selected speakers from the LibriTTS dataset. Defaults to selected_speakers_libri_ids. + hifi_speakers (List[str], optional): The selected speakers from the HiFiTTS dataset. Defaults to selected_speakers_hi_fi_ids. + """ + lang_map = get_lang_map(lang) + processing_lang_type = lang_map.processing_lang_type + self.preprocess_config = PreprocessingConfig( + processing_lang_type, + sampling_rate=sampling_rate, + ) + + self.min_seconds = min_seconds or self.preprocess_config.min_seconds + self.max_seconds = max_seconds or self.preprocess_config.max_seconds + + self.dur_filter = ( + lambda duration: duration >= self.min_seconds + and duration <= self.max_seconds + ) + + self.preprocess_libtts = PreprocessLibriTTS( + self.preprocess_config, + lang, + ) + self.root_dir = Path(root) + self.voicefixer = VoiceFixer() + + # Map the speaker ids to string and list of selected speaker ids to set + self.selected_speakers_libri_ids_ = set(libri_speakers) + self.selected_speakers_hi_fi_ids_ = set(hifi_speakers) + + self.cache = cache + self.cache_dir = Path(cache_dir) / f"cache-{hifitts_path}-{libritts_path}" + + # Prepare the HiFiTTS dataset + self.hifitts_path = self.root_dir / hifitts_path + hifi_cutset_file_path = self.root_dir / hifi_cutset_file_name + + # Initialize the cutset + self.cutset = CutSet() + + # Check if the HiFiTTS dataset has been prepared + if hifi_cutset_file_path.exists(): + self.cutset_hifi = CutSet.from_file(hifi_cutset_file_path) + else: + hifitts_root = hifitts.download_hifitts(self.hifitts_path) + prepared_hifi = hifitts.prepare_hifitts( + hifitts_root, + num_jobs=num_jobs, + ) + + # Add the recordings and supervisions to the CutSet + self.cutset_hifi = prep_2_cutset(prepared_hifi) + # Save the prepared HiFiTTS dataset cutset + self.cutset_hifi.to_file(hifi_cutset_file_path) + + # Filter the HiFiTTS cutset to only include the selected speakers + self.cutset_hifi = self.cutset_hifi.filter( + lambda cut: isinstance(cut, MonoCut) + and str(cut.supervisions[0].speaker) in self.selected_speakers_hi_fi_ids_ + and self.dur_filter(cut.duration), + ).to_eager() + + # Add the HiFiTTS cutset to the final cutset + self.cutset += self.cutset_hifi + + if include_libri: + # Prepare the LibriTTS dataset + self.libritts_path = self.root_dir / libritts_path + libritts_cutset_file_path = self.root_dir / libritts_cutset_file_name + + # Check if the LibriTTS dataset has been prepared + if libritts_cutset_file_path.exists(): + self.cutset_libri = CutSet.from_file(libritts_cutset_file_path) + else: + libritts_root = libritts.download_librittsr( + self.libritts_path, + dataset_parts=libritts_subsets, + ) + prepared_libri = libritts.prepare_librittsr( + libritts_root / "LibriTTS_R", + dataset_parts=libritts_subsets, + num_jobs=num_jobs, + ) + + # Add the recordings and supervisions to the CutSet + self.cutset_libri = prep_2_cutset(prepared_libri) + # Save the prepared cutset for LibriTTS + self.cutset_libri.to_file(libritts_cutset_file_path) + + # Filter the libri cutset to only include the selected speakers + self.cutset_libri = self.cutset_libri.filter( + lambda cut: isinstance(cut, MonoCut) + and str(cut.supervisions[0].speaker) + in self.selected_speakers_libri_ids_ + and self.dur_filter(cut.duration), + ).to_eager() + + # Add the LibriTTS cutset to the final cutset + self.cutset += self.cutset_libri + + # to_eager() is used to evaluates all lazy operations on this manifest + self.cutset = self.cutset.to_eager() + + def get_cache_subdir_path(self, idx: int) -> Path: + r"""Calculate the path to the cache subdirectory. + + Args: + idx (int): The index of the cache subdirectory. + + Returns: + Path: The path to the cache subdirectory. + """ + return self.cache_dir / str(((idx // 1000) + 1) * 1000) + + def get_cache_file_path(self, idx: int) -> Path: + r"""Calculate the path to the cache file. + + Args: + idx (int): The index of the cache file. + + Returns: + Path: The path to the cache file. + """ + return self.get_cache_subdir_path(idx) / f"{idx}.pt" + + def __len__(self) -> int: + r"""Returns the length of the dataset. + + Returns: + int: The length of the dataset. + """ + return len(self.cutset) + + def __getitem__(self, idx: int) -> HifiLibriItem: + r"""Returns the item at the specified index. + + Args: + idx (int): The index of the item. + + Returns: + HifiLibriItem: The item at the specified index. + """ + cache_file = self.get_cache_file_path(idx) + + if self.cache and cache_file.exists(): + cached_data: Dict = torch.load(cache_file) + # Cast the cached data to the PreprocessForAcousticResult class + result = HifiLibriItem(**cached_data) + return result + + cutset = self.cutset[idx] + + if isinstance(cutset, MonoCut) and cutset.recording is not None: + dataset_speaker_id = str(cutset.supervisions[0].speaker) + + # Map the dataset speaker id to the speaker id in the model + speaker_id = selected_speakers_ids.get( + dataset_speaker_id, + len(selected_speakers_ids) + 1, + ) + + # Run voicefixer only for the libri speakers + if str(dataset_speaker_id) in self.selected_speakers_libri_ids_: + audio_path = cutset.recording.sources[0].source + # Restore LibriTTS-R audio + with tempfile.NamedTemporaryFile( + suffix=".wav", + delete=True, + ) as out_file: + self.voicefixer.restore( + input=audio_path, # low quality .wav/.flac file + output=out_file.name, # save file path + cuda=False, # GPU acceleration + mode=0, + ) + audio, _ = sf.read(out_file.name) + # Convert the np audio to a tensor + audio = torch.from_numpy(audio).float().unsqueeze(0) + else: + # Load the audio from the cutset + audio = torch.from_numpy(cutset.load_audio()) + + text: str = str(cutset.supervisions[0].text) + + fileid = str(cutset.supervisions[0].recording_id) + + split_fileid = fileid.split("_") + chapter_id = split_fileid[1] + utterance_id = split_fileid[-1] + + libri_row = ( + audio, + cutset.sampling_rate, + text, + text, + speaker_id, + chapter_id, + utterance_id, + ) + data = self.preprocess_libtts.acoustic(libri_row) + + if data is None: + rand_idx = int( + torch.randint( + 0, + self.__len__(), + (1,), + ).item(), + ) + return self.__getitem__(rand_idx) + + data.wav = data.wav.unsqueeze(0) + + result = HifiLibriItem( + id=data.utterance_id, + wav=data.wav, + mel=data.mel, + pitch=data.pitch, + text=data.phones, + attn_prior=data.attn_prior, + energy=data.energy, + raw_text=data.raw_text, + normalized_text=data.normalized_text, + speaker=speaker_id, + pitch_is_normalized=data.pitch_is_normalized, + lang=lang2id["en"], + dataset_type="hifitts" if idx < len(self.cutset_hifi) else "libritts", + ) + + if self.cache: + # Create the cache subdirectory if it doesn't exist + Path.mkdir( + self.get_cache_subdir_path(idx), + parents=True, + exist_ok=True, + ) + # Save the preprocessed data to the cache + torch.save(asdict(result), cache_file) + + return result + else: + raise FileNotFoundError(f"Cut not found at index {idx}.") + + def __iter__(self): + r"""Method makes the class iterable. It iterates over the `_walker` attribute + and for each item, it gets the corresponding item from the dataset using the + `__getitem__` method. + + Yields: + The item from the dataset corresponding to the current item in `_walker`. + """ + for item in range(self.__len__()): + yield self.__getitem__(item) + + def collate_fn(self, data: List[HifiLibriItem]) -> List: + r"""Collates a batch of data samples. + + Args: + data (List[HifiLibriItem]): A list of data samples. + + Returns: + List: A list of reprocessed data batches. + """ + data_size = len(data) + + idxs = list(range(data_size)) + + # Initialize empty lists to store extracted values + empty_lists: List[List] = [[] for _ in range(12)] + ( + ids, + speakers, + texts, + raw_texts, + mels, + pitches, + attn_priors, + langs, + src_lens, + mel_lens, + wavs, + energy, + ) = empty_lists + + # Extract fields from data dictionary and populate the lists + for idx in idxs: + data_entry = data[idx] + ids.append(data_entry.id) + speakers.append(data_entry.speaker) + texts.append(data_entry.text) + raw_texts.append(data_entry.raw_text) + mels.append(data_entry.mel) + pitches.append(data_entry.pitch) + attn_priors.append(data_entry.attn_prior) + langs.append(data_entry.lang) + src_lens.append(data_entry.text.shape[0]) + mel_lens.append(data_entry.mel.shape[1]) + wavs.append(data_entry.wav) + energy.append(data_entry.energy) + + # Convert langs, src_lens, and mel_lens to numpy arrays + langs = np.array(langs) + src_lens = np.array(src_lens) + mel_lens = np.array(mel_lens) + + # NOTE: Instead of the pitches for the whole dataset, used stat for the batch + # Take only min and max values for pitch + pitches_stat = list(self.normalize_pitch(pitches)[:2]) + + texts = pad_1D(texts) + mels = pad_2D(mels) + pitches = pad_1D(pitches) + attn_priors = pad_3D(attn_priors, len(idxs), max(src_lens), max(mel_lens)) + + speakers = np.repeat( + np.expand_dims(np.array(speakers), axis=1), + texts.shape[1], + axis=1, + ) + langs = np.repeat( + np.expand_dims(np.array(langs), axis=1), + texts.shape[1], + axis=1, + ) + + wavs = pad_2D(wavs) + energy = pad_2D(energy) + + return [ + ids, + raw_texts, + torch.from_numpy(speakers), + texts.int(), + torch.from_numpy(src_lens), + mels, + pitches, + pitches_stat, + torch.from_numpy(mel_lens), + torch.from_numpy(langs), + attn_priors, + wavs, + energy, + ] + + def normalize_pitch( + self, + pitches: List[torch.Tensor], + ) -> Tuple[float, float, float, float]: + r"""Normalizes the pitch values. + + Args: + pitches (List[torch.Tensor]): A list of pitch values. + + Returns: + Tuple: A tuple containing the normalized pitch values. + """ + pitches_t = torch.concatenate(pitches) + + min_value = torch.min(pitches_t).item() + max_value = torch.max(pitches_t).item() + + mean = torch.mean(pitches_t).item() + std = torch.std(pitches_t).item() + + return min_value, max_value, mean, std + + +def train_dataloader( + batch_size: int = 6, + num_workers: int = 5, + sampling_rate: int = 22050, + shuffle: bool = False, + lang: str = "en", + root: str = "datasets_cache", + hifitts_path: str = "hifitts", + hifi_cutset_file_name: str = "hifi.json.gz", + libritts_path: str = "librittsr", + libritts_cutset_file_name: str = "libri.json.gz", + libritts_subsets: List[str] | str = "all", + cache: bool = False, + cache_dir: str = "/dev/shm", + include_libri: bool = True, + libri_speakers: List[str] = speakers_libri_ids, + hifi_speakers: List[str] = speakers_hifi_ids, +) -> DataLoader: + r"""Returns the training dataloader, that is using the HifiLibriDataset dataset. + + Args: + batch_size (int): The batch size. + num_workers (int): The number of workers. + sampling_rate (int): The sampling rate of the audio. Defaults to 22050. + shuffle (bool): Whether to shuffle the dataset. + lang (str): The language of the dataset. + root (str): The root directory of the dataset. + hifitts_path (str): The path to the HiFiTTS dataset. + hifi_cutset_file_name (str): The file name of the HiFiTTS cutset. + libritts_path (str): The path to the LibriTTS dataset. + libritts_cutset_file_name (str): The file name of the LibriTTS cutset. + libritts_subsets (List[str] | str): The subsets of the LibriTTS dataset to use. + cache (bool): Whether to cache the dataset. + cache_dir (str): The directory to cache the dataset in. + include_libri (bool): Whether to include the LibriTTS dataset. + libri_speakers (List[str]): The selected speakers from the LibriTTS dataset. + hifi_speakers (List[str]): The selected speakers from the HiFiTTS dataset. + + Returns: + DataLoader: The training dataloader. + """ + dataset = HifiLibriDataset( + root=root, + hifitts_path=hifitts_path, + sampling_rate=sampling_rate, + hifi_cutset_file_name=hifi_cutset_file_name, + libritts_path=libritts_path, + libritts_cutset_file_name=libritts_cutset_file_name, + libritts_subsets=libritts_subsets, + cache=cache, + cache_dir=cache_dir, + lang=lang, + include_libri=include_libri, + libri_speakers=libri_speakers, + hifi_speakers=hifi_speakers, + ) + + train_loader = DataLoader( + dataset, + # 4x80Gb max 10 sec audio + # batch_size=20, # self.train_config.batch_size, + # 4*80Gb max ~20.4 sec audio + batch_size=batch_size, + # TODO: find the optimal num_workers + num_workers=num_workers, + persistent_workers=True, + pin_memory=True, + shuffle=shuffle, + collate_fn=dataset.collate_fn, + ) + + return train_loader diff --git a/training/datasets/lhotse_notebook.py b/training/datasets/lhotse_notebook.py new file mode 100644 index 0000000000000000000000000000000000000000..ad5f8732c5de3637cbfce58e3a25a0bf42b3fca4 --- /dev/null +++ b/training/datasets/lhotse_notebook.py @@ -0,0 +1,543 @@ +# %% +import os +from pathlib import Path +from pprint import pprint + +from lhotse.recipes import ( + download_voxceleb1, + download_voxceleb2, + hifitts, + libritts, + prepare_voxceleb, +) +import pandas as pd + +# %% +root_dir = Path("../../datasets_cache") +# root_dir = Path("datasets_cache") + +voxceleb1_path = root_dir / "voxceleb1" +voxceleb2_path = root_dir / "voxceleb2" + +hifitts_path = root_dir / "hifitts" +libritts_path = root_dir / "librittsr" + +num_jobs = os.cpu_count() - 3 # type: ignore + +num_jobs, hifitts_path + +# %% +# voxceleb1_root = download_voxceleb1(voxceleb1_path) +# voxceleb1_root + +# %% +# voxceleb2_root = download_voxceleb2(voxceleb2_path) +# voxceleb2_root + +# %% +hifitts_root = hifitts.download_hifitts(hifitts_path) +hifitts_root + + +# %% +result = hifitts.prepare_hifitts(hifitts_root, num_jobs=num_jobs) +result + + +# %% +result.keys() + +# %% +from lhotse import CutSet, Fbank, FbankConfig, Mfcc, MfccConfig, RecordingSet + +cuts_train = CutSet.from_manifests(**result["6670_other_test"]) # type: ignore +cuts_train + +# %% +pprint(cuts_train[0]) + +# %% +from lhotse.cut import Cut + +# Filter the CutSet to only include cuts that are no more than the duration limit +duration_limit_min = 2.0 +duration_limit_max = 2.5 +# Duration limit in seconds +cuts_train = cuts_train.filter( + lambda cut: isinstance(cut, Cut) + and cut.duration >= duration_limit_min + and cut.duration <= duration_limit_max, +) +cuts_train + +# %% +cuts_train[0].supervisions[0] + +# %% +# filter_length=2048, +# hop_length=512, # NOTE: 441 ?? https://github.com/jik876/hifi-gan/issues/116#issuecomment-1436999858 +# win_length=2048, +# n_mel_channels=128, +# mel_fmin=20, +# mel_fmax=11025, + +fbank = Fbank( + FbankConfig( + sampling_rate=44100, + num_filters=128, + ), +) + +cuts_train_fbank = cuts_train.compute_and_store_features( + extractor=fbank, + storage_path=hifitts_root / "features", + num_jobs=1, +) + +cuts_train_fbank + +# %% +# cuts_train_fbank.to_file(hifitts_root / "cuts_train.json.gz") + +# %% +cuts_train_fbank[0].plot_features() + +# %% +cuts_train_fbank_item = cuts_train_fbank[0] +cuts_train_fbank_item + +# %% +from lhotse.cut import MonoCut + +if isinstance(cuts_train_fbank_item, MonoCut): + print(cuts_train_fbank_item.features) + +# %% +cuts_train_fbank_item.plot_audio() + +# %% +cuts_train_fbank_item.play_audio() + +# %% +from lhotse import CutSet +from lhotse.dataset import ( + SimpleCutSampler, + UnsupervisedDataset, + UnsupervisedWaveformDataset, +) +from torch.utils.data import DataLoader, Dataset + +dataset = UnsupervisedDataset() +sampler = SimpleCutSampler(cuts_train_fbank, max_duration=300) + +dataloader = DataLoader(dataset, sampler=sampler, batch_size=None) + +batch = next(iter(dataloader)) +batch + + +# %% +batch["cuts"][0].recording.sources[0].load_audio().shape + +# %% +batch["cuts"][0].features + +# %% +batch["features"][0].shape + +# %% +batch["features"][0] + +# %% +# Prepare the LibriTTS dataset +libritts_root = libritts.download_librittsr( + libritts_path, + dataset_parts=["train-clean-100"], +) +libritts_root, libritts_path + +# %% +prepared_libri = libritts.prepare_librittsr( + libritts_root / "LibriTTS_R", + # dataset_parts=["dev-clean"], + dataset_parts=["train-clean-100"], + num_jobs=num_jobs, +) + +# %% +prepared_libri + +# %% +prepared_libri_100 = ( + pd.DataFrame(prepared_libri["train-clean-100"]["supervisions"]) + .groupby("speaker")["duration"] + .sum() + .sort_values(ascending=False) +) +prepared_libri_100 + +# %% +for k in prepared_libri: + prepared_libri_ = ( + pd.DataFrame(prepared_libri[k]["supervisions"]) + .groupby("speaker")["duration"] + .sum() + .sort_values(ascending=False) + ) + print(prepared_libri_.loc[prepared_libri_ >= 1800]) + +# %% +from lhotse import CutSet, SupervisionSet + +supervisions_libri = SupervisionSet() + +supervisions_libri.to_file(libritts_root / "supervisions_libri.json.gz") + +# dev-clean +# Series([], Name: duration, dtype: float64) +# dev-other +# Series([], Name: duration, dtype: float64) +# test-clean +# speaker +# 3570 1865.052667 +# Name: duration, dtype: float64 +# test-other +# Series([], Name: duration, dtype: float64) +# train-clean-100 +# speaker +# 40 2096.569333 +# 6209 1926.765000 +# 7447 1915.213333 +# 1088 1900.926000 +# Name: duration, dtype: float64 +# train-clean-360 +# speaker +# 3003 2385.213333 +# 2204 2242.730333 +# 3307 2086.246500 +# 8080 2051.131500 +# 5935 1959.650833 +# 3922 1938.523500 +# 7982 1893.050833 +# 3638 1843.324000 +# 3032 1812.692000 +# Name: duration, dtype: float64 +# train-other-500 +# speaker +# 215 2385.047833 +# 6594 2341.286667 +# 3433 2206.806500 +# 3867 2118.326167 +# 5733 2097.689833 +# 7649 2016.925500 +# 2834 2008.083000 +# 8291 1977.892000 +# 483 1964.766000 +# 5181 1959.280000 +# 8799 1909.690500 +# 7839 1888.650500 +# 1665 1877.726833 +# 8430 1872.845500 +# 47 1861.966167 +# 2361 1839.646333 +# 1132 1838.686333 +# 5439 1837.487000 +# 3319 1821.083833 +# 5445 1808.444667 +# 2208 1804.525833 +# 8346 1804.405500 +# Name: duration, dtype: float64 + +selected_speakers_man = [ + # train-clean-100 + "40", + "1088", + # train-clean-360 + "3307", + "5935", + "3032", + # train-other-500 + "215", + "6594", + "3867", + "5733", + "8291", + "5181", + "8799", + "2361", + "1132", + "5439", + "3319", + "8346", +] + + +# %% +num_speakers_lib_100_over_1900_sec = prepared_libri_100.loc[prepared_libri_100 >= 1900] +num_speakers_lib_100_over_1900_sec + +# %% +prepared_libri_360 = libritts.prepare_librittsr( + libritts_root / "LibriTTS_R", + # dataset_parts=["dev-clean"], + dataset_parts=["train-clean-360"], + num_jobs=num_jobs, +) + +# %% +speaker_durations_360 = ( + pd.DataFrame(prepared_libri_360["train-clean-360"]["supervisions"]) + .groupby("speaker")["duration"] + .sum() + .sort_values(ascending=False) +) +speaker_durations_360 + +# %% +# Get the speaker IDs from both dataframes +speaker_ids_100 = prepared_libri_100.index +speaker_ids_360 = speaker_durations_360.index + +# Find the intersection of the speaker IDs +common_speaker_ids = speaker_ids_100.intersection(speaker_ids_360) + +# No intersection! +common_speaker_ids + +# %% +num_speakers_lib_360_over_1900_sec = speaker_durations_360.loc[ + speaker_durations_360 > 1900 +].count() +num_speakers_lib_360_over_1900_sec + +# %% +from lhotse import CutSet, Fbank, FbankConfig + +cuts_train = CutSet.from_manifests(**prepared_libri["train-clean-100"]) # type: ignore +cuts_train + +# %% +# You can save the prepared CutSet to a file! +cuts_train.to_file("./libri_selected.json.gz") + +cuts_train.to_file(root_dir / "./libri_selected.json.gz") + +# %% +from lhotse import CutSet, SupervisionSet + +libri_selected = CutSet.from_file(root_dir / "libri.json.gz") +libri_selected + +# %% + +pprint(libri_selected[0]) + +print(libri_selected[0].recording.sources[0].source) + +# %% +libri_selected[0].play_audio() + +# %% +import torchaudio + +torchaudio.load( + "datasets_cache/librittsr/LibriTTS_R/dev-clean/5694/64025/5694_64025_000017_000002.wav", +) + +# %% +supervisions_libri = SupervisionSet.from_file( + root_dir / "supervisions_libri.json.gz", +) +recordings_libri = RecordingSet.from_file( + root_dir / "recordings_libri.json.gz", +) +supervisions_libri, recordings_libri + + +# %% +supervisions_libri[0] + +# %% +speakers_dur = ( + pd.DataFrame(supervisions_libri) + .groupby("speaker")["duration"] + .sum() + .sort_values(ascending=False) +) + +# %% +speakers_dur_1900 = speakers_dur.loc[speakers_dur >= 1900] +speakers_dur_1900 + +# %% +# selected_1900_ids = set( +# map(int, speakers_dur_1900.index.to_list()), +# ) + +selected_1900_ids = set( + speakers_dur_1900.index.to_list(), +) +selected_1900_ids + +# %% +duration_limit_min = 0.5 +duration_limit_max = 35.0 + +libri_selected.filter( + lambda cut: isinstance(cut, Cut) + and cut.supervisions[0].speaker in selected_1900_ids + and cut.duration >= duration_limit_min + and cut.duration <= duration_limit_max, +) + +# %% +libri_selected[0] + +# %% +cuts_train_frame = pd.DataFrame(cuts_train) +cuts_train_frame + +# %% +cuts_train[0].supervisions[0].speaker + +# %% +# duration_limit_min = 2.0 +# duration_limit_max = 2.5 + +cuts_train = cuts_train.filter( + lambda cut: isinstance(cut, Cut) and cut.supervisions[0].speaker == "5338", + # and cut.duration >= duration_limit_min + # and cut.duration <= duration_limit_max, +) + +cuts_train + +# %% +# cuts_train.map(lambda cut: cut.supervisions[0].speaker) +# %% +cuts_train[0] + +# %% +len(cuts_train) + +# %% +selected_speakers_libri_ids = [ + # train-clean-100 + 40, + 1088, + # train-clean-360 + 3307, + 5935, + 3032, + # train-other-500 + 215, + 6594, + 3867, + 5733, + 8291, + 5181, + 8799, + 2361, + 1132, + 5439, + 3319, + 8346, +] + +# The selected speakers from the HiFiTTS dataset +selected_speakers_hi_fi_ids = [ + 92, + 6670, + 6671, + 6097, + 8051, + 11614, + 11697, + 9017, + 12787, + 9136, +] + +selected_speakers_ids = { + v: k + for k, v in enumerate( + selected_speakers_libri_ids + selected_speakers_hi_fi_ids, + ) +} + +selected_speakers_ids[1088] + +# %% + +selected_speakers_libri_ids = [ + # train-clean-100 + 40, + 1088, + # train-clean-360 + 3307, + 5935, + 3032, + # train-other-500 + 215, + 6594, + 3867, + 5733, + 8291, + 5181, + 8799, + 2361, + 1132, + 5439, + 3319, + 8346, +] + +# The selected speakers from the HiFiTTS dataset +selected_speakers_hi_fi_ids = [ + "Cori Samuel", # 92, + "Phil Benson", # 6097, + "Mike Pelton", # 6670, + "Tony Oliva", # 6671, + "Maria Kasper", # 8051, + "John Van Stan", # 9017, + "Helen Taylor", # 9136, + "Sylviamb", # 11614, + "Celine Major", # 11697, + "LikeManyWaters", # 12787, +] + +# Map the speaker ids to string and list of selected speaker ids to set +selected_speakers_ids = { + v: k + for k, v in enumerate( + selected_speakers_libri_ids + selected_speakers_hi_fi_ids, + ) +} + +selected_speakers_ids, len(selected_speakers_ids) + +# %% +import os +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + +from pathlib import Path + +from IPython import display +import torchaudio +from voicefixer import Vocoder + +from .hifi_libri_dataset import HifiLibriDataset, HifiLibriItem + +vocoder_vf = Vocoder(44100) +dataset = HifiLibriDataset(cache_dir="datasets_cache", cache=True) + +item = dataset[0] +wav = vocoder_vf.forward(item.mel.permute((1, 0)).unsqueeze(0)) +display.Audio(wav.squeeze(0).cpu().detach().numpy(), rate=44100) +# wav_path = Path(f"results/{item.id}.wav") +# torchaudio.save(str(wav_path), wav, 44100) + +# %% diff --git a/training/datasets/libritts_dataset_acoustic.py b/training/datasets/libritts_dataset_acoustic.py new file mode 100644 index 0000000000000000000000000000000000000000..045e97066606409bbb669a0b0bba0b036ee3e21b --- /dev/null +++ b/training/datasets/libritts_dataset_acoustic.py @@ -0,0 +1,268 @@ +import json +import os +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +import torch +from torch.utils.data import Dataset + +from models.config import PreprocessingConfigUnivNet as PreprocessingConfig +from models.config import get_lang_map, lang2id +from training.preprocess import PreprocessLibriTTS +from training.tools import pad_1D, pad_2D, pad_3D + +from .libritts_r import LIBRITTS_R + + +class LibriTTSDatasetAcoustic(Dataset): + r"""Loading preprocessed acoustic model data.""" + + def __init__( + self, + lang: str = "en", + root: str = "datasets_cache/LIBRITTS", + url: str = "train-clean-360", + download: bool = False, + cache: bool = False, + mem_cache: bool = False, + cache_dir: str = "datasets_cache", + selected_speaker_ids: Optional[List[int]] = None, + ): + r"""A PyTorch dataset for loading preprocessed acoustic data. + + Args: + root (str): Path to the directory where the dataset is found or downloaded. + lang (str): The language of the dataset. + url (str): The dataset url, default "train-clean-360". + download (bool, optional): Whether to download the dataset if it is not found. Defaults to True. + cache (bool, optional): Whether to cache the preprocessed data to RAM. Defaults to False. + mem_cache (bool, optional): Whether to cache the preprocessed data. Defaults to False. + cache_dir (str, optional): Path to the directory where the cache is stored. Defaults to "datasets_cache". + selected_speaker_ids (Optional[List[int]], optional): A list of selected speakers. Defaults to None. + """ + lang_map = get_lang_map(lang) + processing_lang_type = lang_map.processing_lang_type + preprocess_config = PreprocessingConfig(processing_lang_type) + + self.dataset = LIBRITTS_R( + root=root, + download=download, + url=url, + selected_speaker_ids=selected_speaker_ids, + min_audio_length=preprocess_config.min_seconds, + max_audio_length=preprocess_config.max_seconds, + ) + self.cache = cache + + # Calculate the directory for the cache file + self.cache_subdir = lambda idx: str(((idx // 1000) + 1) * 1000) + + self.cache_dir = os.path.join(cache_dir, f"cache-{url}") + + self.mem_cache = mem_cache + self.memory_cache = {} + + # Load the id_mapping dictionary from the JSON file + with open("speaker_id_mapping_libri.json") as f: + self.id_mapping = json.load(f) + + self.preprocess_libtts = PreprocessLibriTTS( + preprocess_config, + lang, + ) + + def __len__(self) -> int: + r"""Returns the number of samples in the dataset. + + Returns + int: Number of samples in the dataset. + """ + return len(self.dataset) + + def __getitem__(self, idx: int) -> Dict[str, Any]: + r"""Returns a sample from the dataset at the given index. + + Args: + idx (int): Index of the sample to return. + + Returns: + Dict[str, Any]: A dictionary containing the sample data. + """ + # Check if the data is in the memory cache + if self.mem_cache and idx in self.memory_cache: + return self.memory_cache[idx] + + # Check if the data is in the cache + cache_subdir_path = os.path.join(self.cache_dir, self.cache_subdir(idx)) + cache_file = os.path.join(cache_subdir_path, f"{idx}.pt") + + # Check if the data is in the cache + if self.cache and os.path.exists(cache_file): + # If the data is in the cache, load it from the cache file and return it + data = torch.load(cache_file) + return data + + # Retrive the dataset row + data = self.dataset[idx] + + data = self.preprocess_libtts.acoustic(data) + + # TODO: bad way to do filtering, fix this! + if data is None: + # print("Skipping due to preprocessing error") + rand_idx = np.random.randint(0, self.__len__()) + return self.__getitem__(rand_idx) + + data.wav = data.wav.unsqueeze(0) + + result = { + "id": data.utterance_id, + "wav": data.wav, + "mel": data.mel, + "pitch": data.pitch, + "text": data.phones, + "attn_prior": data.attn_prior, + "energy": data.energy, + "raw_text": data.raw_text, + "normalized_text": data.normalized_text, + "speaker": self.id_mapping.get(str(data.speaker_id)), + "pitch_is_normalized": data.pitch_is_normalized, + # TODO: fix lang! + "lang": lang2id["en"], + } + + # Add the data to the memory cache + if self.mem_cache: + self.memory_cache[idx] = result + + if self.cache: + # Create the cache subdirectory if it doesn't exist + os.makedirs(cache_subdir_path, exist_ok=True) + + # Save the preprocessed data to the cache + torch.save(result, cache_file) + + return result + + def __iter__(self): + r"""Method makes the class iterable. It iterates over the `_walker` attribute + and for each item, it gets the corresponding item from the dataset using the + `__getitem__` method. + + Yields: + The item from the dataset corresponding to the current item in `_walker`. + """ + for item in range(self.__len__()): + yield self.__getitem__(item) + + def collate_fn(self, data: List) -> List: + r"""Collates a batch of data samples. + + Args: + data (List): A list of data samples. + + Returns: + List: A list of reprocessed data batches. + """ + data_size = len(data) + + idxs = list(range(data_size)) + + # Initialize empty lists to store extracted values + empty_lists: List[List] = [[] for _ in range(12)] + ( + ids, + speakers, + texts, + raw_texts, + mels, + pitches, + attn_priors, + langs, + src_lens, + mel_lens, + wavs, + energy, + ) = empty_lists + + # Extract fields from data dictionary and populate the lists + for idx in idxs: + data_entry = data[idx] + ids.append(data_entry["id"]) + speakers.append(data_entry["speaker"]) + texts.append(data_entry["text"]) + raw_texts.append(data_entry["raw_text"]) + mels.append(data_entry["mel"]) + pitches.append(data_entry["pitch"]) + attn_priors.append(data_entry["attn_prior"]) + langs.append(data_entry["lang"]) + src_lens.append(data_entry["text"].shape[0]) + mel_lens.append(data_entry["mel"].shape[1]) + wavs.append(data_entry["wav"]) + energy.append(data_entry["energy"]) + + # Convert langs, src_lens, and mel_lens to numpy arrays + langs = np.array(langs) + src_lens = np.array(src_lens) + mel_lens = np.array(mel_lens) + + # NOTE: Instead of the pitches for the whole dataset, used stat for the batch + # Take only min and max values for pitch + pitches_stat = list(self.normalize_pitch(pitches)[:2]) + + texts = pad_1D(texts) + mels = pad_2D(mels) + pitches = pad_1D(pitches) + attn_priors = pad_3D(attn_priors, len(idxs), max(src_lens), max(mel_lens)) + + speakers = np.repeat( + np.expand_dims(np.array(speakers), axis=1), + texts.shape[1], + axis=1, + ) + langs = np.repeat( + np.expand_dims(np.array(langs), axis=1), + texts.shape[1], + axis=1, + ) + + wavs = pad_2D(wavs) + energy = pad_2D(energy) + + return [ + ids, + raw_texts, + torch.from_numpy(speakers), + texts.int(), + torch.from_numpy(src_lens), + mels, + pitches, + pitches_stat, + torch.from_numpy(mel_lens), + torch.from_numpy(langs), + attn_priors, + wavs, + energy, + ] + + def normalize_pitch( + self, + pitches: List[torch.Tensor], + ) -> Tuple[float, float, float, float]: + r"""Normalizes the pitch values. + + Args: + pitches (List[torch.Tensor]): A list of pitch values. + + Returns: + Tuple: A tuple containing the normalized pitch values. + """ + pitches_t = torch.concatenate(pitches) + + min_value = torch.min(pitches_t).item() + max_value = torch.max(pitches_t).item() + + mean = torch.mean(pitches_t).item() + std = torch.std(pitches_t).item() + + return min_value, max_value, mean, std diff --git a/training/datasets/libritts_dataset_vocoder.py b/training/datasets/libritts_dataset_vocoder.py new file mode 100644 index 0000000000000000000000000000000000000000..72718c43b9e64568e1a8409654257b175b15115b --- /dev/null +++ b/training/datasets/libritts_dataset_vocoder.py @@ -0,0 +1,114 @@ +from typing import Any, Dict, List + +import numpy as np +import torch +from torch.utils.data import Dataset +from torchaudio import datasets + +from models.config import PreprocessingConfigUnivNet, get_lang_map +from training.preprocess import PreprocessLibriTTS +from training.tools import pad_1D, pad_2D + + +class LibriTTSDatasetVocoder(Dataset): + r"""Loading preprocessed univnet model data.""" + + def __init__( + self, + root: str, + batch_size: int, + download: bool = True, + lang: str = "en", + ): + r"""A PyTorch dataset for loading preprocessed univnet data. + + Args: + root (str): Path to the directory where the dataset is found or downloaded. + batch_size (int): Batch size for the dataset. + download (bool, optional): Whether to download the dataset if it is not found. Defaults to True. + """ + self.dataset = datasets.LIBRITTS(root=root, download=download) + self.batch_size = batch_size + + lang_map = get_lang_map(lang) + self.preprocess_libtts = PreprocessLibriTTS( + PreprocessingConfigUnivNet(lang_map.processing_lang_type), + ) + + def __len__(self) -> int: + r"""Returns the number of samples in the dataset. + + Returns + int: Number of samples in the dataset. + """ + return len(self.dataset) + + def __getitem__(self, idx: int) -> Dict[str, Any]: + r"""Returns a sample from the dataset at the given index. + + Args: + idx (int): Index of the sample to return. + + Returns: + Dict[str, Any]: A dictionary containing the sample data. + """ + # Retrive the dataset row + data = self.dataset[idx] + + data = self.preprocess_libtts.univnet(data) + + if data is None: + # print("Skipping due to preprocessing error") + rand_idx = np.random.randint(0, self.__len__()) + return self.__getitem__(rand_idx) + + mel, audio, speaker_id = data + + return { + "mel": mel, + "audio": audio, + "speaker_id": speaker_id, + } + + def collate_fn(self, data: List) -> List: + r"""Collates a batch of data samples. + + Args: + data (List): A list of data samples. + + Returns: + List: A list of reprocessed data batches. + """ + data_size = len(data) + + idxs = list(range(data_size)) + + # Initialize empty lists to store extracted values + empty_lists: List[List] = [[] for _ in range(4)] + ( + mels, + mel_lens, + audios, + speaker_ids, + ) = empty_lists + + # Extract fields from data dictionary and populate the lists + for idx in idxs: + data_entry = data[idx] + + mels.append(data_entry["mel"]) + mel_lens.append(data_entry["mel"].shape[1]) + audios.append(data_entry["audio"]) + speaker_ids.append(data_entry["speaker_id"]) + + mels = torch.tensor(pad_2D(mels), dtype=torch.float32) + mel_lens = torch.tensor(mel_lens, dtype=torch.int64) + audios = torch.tensor(pad_1D(audios), dtype=torch.float32) + speaker_ids = torch.tensor(speaker_ids, dtype=torch.int64) + + return [ + mels, + mel_lens, + audios, + speaker_ids, + ] diff --git a/training/datasets/libritts_mm_dataset_acoustic.py b/training/datasets/libritts_mm_dataset_acoustic.py new file mode 100644 index 0000000000000000000000000000000000000000..797c21e7d390b8c2a3c065ab9f4ea43189e2065b --- /dev/null +++ b/training/datasets/libritts_mm_dataset_acoustic.py @@ -0,0 +1,140 @@ +from typing import Any, List, Tuple + +import numpy as np +import torch +from torch.utils.data import Dataset + +from training.tools import pad_1D, pad_2D, pad_3D + + +class LibriTTSMMDatasetAcoustic(Dataset): + def __init__(self, file_path: str): + r"""A PyTorch dataset for loading preprocessed acoustic data stored in memory-mapped files. + + Args: + file_path (str): Path to the memory-mapped file. + """ + self.data = torch.load(file_path) + + def __getitem__(self, idx: int): + r"""Returns a sample from the dataset at the given index. + + Args: + idx (int): Index of the sample to return. + + Returns: + Dict[str, Any]: A dictionary containing the sample data. + """ + return self.data[idx] + + def __len__(self): + r"""Returns the number of samples in the dataset. + + Returns + int: Number of samples in the dataset. + """ + return len(self.data) + + def collate_fn(self, data: List) -> List: + r"""Collates a batch of data samples. + + Args: + data (List): A list of data samples. + + Returns: + List: A list of reprocessed data batches. + """ + data_size = len(data) + + idxs = list(range(data_size)) + + # Initialize empty lists to store extracted values + empty_lists: List[List] = [[] for _ in range(11)] + ( + ids, + speakers, + texts, + raw_texts, + mels, + pitches, + attn_priors, + langs, + src_lens, + mel_lens, + wavs, + ) = empty_lists + + # Extract fields from data dictionary and populate the lists + for idx in idxs: + data_entry = data[idx] + ids.append(data_entry["id"]) + speakers.append(data_entry["speaker"]) + texts.append(data_entry["text"]) + raw_texts.append(data_entry["raw_text"]) + mels.append(data_entry["mel"]) + pitches.append(data_entry["pitch"]) + attn_priors.append(data_entry["attn_prior"].numpy()) + langs.append(data_entry["lang"]) + src_lens.append(data_entry["text"].shape[0]) + mel_lens.append(data_entry["mel"].shape[1]) + wavs.append(data_entry["wav"].numpy()) + + # Convert langs, src_lens, and mel_lens to numpy arrays + langs = np.array(langs) + src_lens = np.array(src_lens) + mel_lens = np.array(mel_lens) + + # NOTE: Instead of the pitches for the whole dataset, used stat for the batch + # Take only min and max values for pitch + pitches_stat = list(self.normalize_pitch(pitches)[:2]) + + texts = pad_1D(texts) + mels = pad_2D(mels) + pitches = pad_1D(pitches) + attn_priors = pad_3D(attn_priors, len(idxs), max(src_lens), max(mel_lens)) + + speakers = np.repeat( + np.expand_dims(np.array(speakers), axis=1), texts.shape[1], axis=1, + ) + langs = np.repeat( + np.expand_dims(np.array(langs), axis=1), texts.shape[1], axis=1, + ) + + wavs = pad_2D(wavs) + + return [ + ids, + raw_texts, + torch.from_numpy(speakers), + torch.from_numpy(texts).int(), + torch.from_numpy(src_lens), + torch.from_numpy(mels), + torch.from_numpy(pitches), + pitches_stat, + torch.from_numpy(mel_lens), + torch.from_numpy(langs), + torch.from_numpy(attn_priors), + torch.from_numpy(wavs), + ] + + def normalize_pitch( + self, pitches: List[torch.Tensor], + ) -> Tuple[float, float, float, float]: + r"""Normalizes the pitch values. + + Args: + pitches (List[torch.Tensor]): A list of pitch values. + + Returns: + Tuple: A tuple containing the normalized pitch values. + """ + pitches_t = torch.concatenate(pitches) + + min_value = torch.min(pitches_t).item() + max_value = torch.max(pitches_t).item() + + mean = torch.mean(pitches_t).item() + std = torch.std(pitches_t).item() + + return min_value, max_value, mean, std + diff --git a/training/datasets/libritts_preprocessed.py b/training/datasets/libritts_preprocessed.py new file mode 100644 index 0000000000000000000000000000000000000000..201d4350cd75d7658590e0a381e332339939477d --- /dev/null +++ b/training/datasets/libritts_preprocessed.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +import os +from typing import Any, List + +import torch +from torch.utils.data import Dataset + + +@dataclass +class PreprocessedData: + id: Any + raw_text: Any + speaker: Any + text: Any + src_len: Any + mel: Any + pitch: Any + pitch_stat: Any + mel_len: Any + lang: Any + attn_prior: Any + wav: Any + energy: Any + + +@dataclass +class PreprocessedDataset(Dataset): + def __init__(self, cache_dir: str = "datasets_cache/LibriTTS_preprocessed"): + self.cache_dir = cache_dir + self.data = [] + + for file in os.listdir(self.cache_dir): + if file.endswith(".pt"): + self.data.extend(torch.load(os.path.join(self.cache_dir, file))) + + for file in self.data_files: + self.data.extend(torch.load(os.path.join(self.cache_dir, file))) + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] diff --git a/training/datasets/libritts_r.py b/training/datasets/libritts_r.py new file mode 100644 index 0000000000000000000000000000000000000000..eec292ccef17cc61815f0ed8eb9ba96862b546c6 --- /dev/null +++ b/training/datasets/libritts_r.py @@ -0,0 +1,267 @@ +from multiprocessing import Pool, cpu_count +import os +from pathlib import Path +from typing import Optional, Tuple, Union + +import pandas as pd +from torch import Tensor +from torch.utils.data import Dataset +import torchaudio +from torchaudio._internal import download_url_to_file # type: ignore +from torchaudio.datasets.utils import _extract_tar + +URL = "train-clean-100" +FOLDER_IN_ARCHIVE = "LibriTTS" +_CHECKSUMS = { + "http://us.openslr.org/resources/141/dev_clean.tar.gz": "2c1f5312914890634cc2d15783032ff3", + "http://us.openslr.org/resources/141/dev_other.tar.gz": "62d3a80ad8a282b6f31b3904f0507e4f", + "http://us.openslr.org/resources/141/test_clean.tar.gz": "4d373d453eb96c0691e598061bbafab7", + "http://us.openslr.org/resources/141/test_other.tar.gz": "dbc0959d8bdb6d52200595cabc9995ae", + "http://us.openslr.org/resources/141/train_clean_100.tar.gz": "6df668d8f5f33e70876bfa33862ad02b", + "http://us.openslr.org/resources/141/train_clean_360.tar.gz": "382eb3e64394b3da6a559f864339b22c", + "http://us.openslr.org/resources/141/train_other_500.tar.gz": "a37a8e9f4fe79d20601639bf23d1add8", +} + + +def load_libritts_item( + fileid: str, + path: str, + ext_audio: str, + ext_original_txt: str, + ext_normalized_txt: str, +) -> Tuple[Tensor, int, str, str, int, int, str]: + speaker_id, chapter_id, segment_id, utterance_id = fileid.split("_") + utterance_id = fileid + + file_audio = utterance_id + ext_audio + file_audio = os.path.join(path, speaker_id, chapter_id, file_audio) + + # Load audio + waveform, sample_rate = torchaudio.load(file_audio) # type: ignore + + # Try to load transcriptions from individual files + normalized_text_filename = utterance_id + ext_normalized_txt + normalized_text_path = os.path.join(path, speaker_id, chapter_id, normalized_text_filename) + + original_text_filename = utterance_id + ext_original_txt + original_text_path = os.path.join(path, speaker_id, chapter_id, original_text_filename) + + try: + # Load normalized text + with open(normalized_text_path) as ft: + normalized_text = ft.readline() + + # Load original text + with open(original_text_path) as ft: + original_text = ft.readline() + + except FileNotFoundError: + # If individual files are not found, load from .tsv file + trans_file = f"{speaker_id}_{chapter_id}.trans.tsv" + trans_file = os.path.join(path, speaker_id, chapter_id, trans_file) + df = pd.read_csv(trans_file, sep="\t", header=None, names=["id", "original_text", "normalized_text"]) + + row = df[df["id"] == utterance_id].iloc[0] + + original_text = row["original_text"] + normalized_text = row["normalized_text"] + + # Save original_text and normalized_text to separate text files + with open(normalized_text_path, "w") as ft: + ft.write(original_text) + + with open(original_text_path, "w") as ft: + ft.write(normalized_text) + + return ( + waveform, + sample_rate, + original_text, + normalized_text, + int(speaker_id), + int(chapter_id), + utterance_id, + ) + + +def check_audio_length(args: Tuple[str, str, str, str, str, float, Optional[float]]) -> Optional[str]: + """Check if the duration of an audio file is within a specified range. + + Args: + args (Tuple[str, str, str, str, str, float, Optional[float]]): A tuple containing the following: + - fileid (str): The ID of the file to check. + - path (str): The path to the directory containing the audio file. + - ext_audio (str): The file extension of the audio file. + - ext_original_txt (str): The file extension of the original text file. + - ext_normalized_txt (str): The file extension of the normalized text file. + - min_audio_length (float): The minimum audio length in seconds. If the audio is shorter than this, it will be excluded. + - max_audio_length (Optional[float]): The maximum audio length in seconds. If the audio is longer than this, it will be excluded. If None, no maximum length is enforced. + + Returns: + Optional[str]: The ID of the file if its duration is within the specified range, or None if it's not. + """ + ( + fileid, + path, + ext_audio, + ext_original_txt, + ext_normalized_txt, + min_audio_length, + max_audio_length, + ) = args + + waveform, sample_rate, _, _, _, _, _ = load_libritts_item( + fileid, + path, + ext_audio, + ext_original_txt, + ext_normalized_txt, + ) + duration = waveform.shape[1] / sample_rate + + min_length_condition = duration > min_audio_length if min_audio_length > 0.0 else True + max_length_condition = duration <= max_audio_length if max_audio_length is not None else True + + if min_length_condition and max_length_condition: + return fileid + else: + return None + + +class LIBRITTS_R(Dataset): + """*LibriTTS-R*: A Restored Multi-Speaker Text-to-Speech Corpus, arXiv, 2023 + + Args: + root (str or Path): Path to the directory where the dataset is found or downloaded. + url (str, optional): The URL to download the dataset from, + or the type of the dataset to dowload. + Allowed type values are ``"dev-clean"``, ``"dev-other"``, ``"test-clean"``, + ``"test-other"``, ``"train-clean-100"``, ``"train-clean-360"`` and + ``"train-other-500"``. (default: ``"train-clean-100"``) + folder_in_archive (str, optional): + The top-level directory of the dataset. (default: ``"LibriTTS"``) + download (bool, optional): + Whether to download the dataset if it is not found at root path. (default: ``False``). + selected_speaker_ids (list, optional): List of speaker IDs to be selected. (default: ``None``) + min_audio_length (float, optional): Minimum audio length in seconds. (default: ``0.0``) + max_audio_length (float, optional): Maximum audio length in seconds. (default: ``None``) + """ + + _ext_original_txt = ".original.txt" + _ext_normalized_txt = ".normalized.txt" + _ext_audio = ".wav" + + def __init__( + self, + root: Union[str, Path], + url: str = URL, + folder_in_archive: str = FOLDER_IN_ARCHIVE, + download: bool = False, + selected_speaker_ids: Union[None, list] = None, + min_audio_length: float = 0.0, + max_audio_length: Union[None, float] = None, + ) -> None: + + if url in [ + "dev-clean", + "dev-other", + "test-clean", + "test-other", + "train-clean-100", + "train-clean-360", + "train-clean-460", + "train-other-500", + "train-960", + ]: + + ext_archive = ".tar.gz" + base_url = "http://us.openslr.org/resources/141/" + + url = os.path.join(base_url, url + ext_archive) + + # Get string representation of 'root' in case Path object is passed + root = os.fspath(root) + + basename = os.path.basename(url) + archive = os.path.join(root, basename) + + basename = basename.split(".")[0] + folder_in_archive = os.path.join(folder_in_archive, basename) + + self._path = os.path.join(root, folder_in_archive) + + if download: + if not os.path.isdir(self._path): + if not os.path.isfile(archive): + checksum = _CHECKSUMS.get(url) + download_url_to_file(url, archive, hash_prefix=checksum) + _extract_tar(archive) + else: + if not os.path.exists(self._path): + raise RuntimeError( + f"The path {self._path} doesn't exist. " + "Please check the ``root`` path or set `download=True` to download it", + ) + + self._walker = sorted(str(p.stem) for p in Path(self._path).glob("*/*/*" + self._ext_audio)) + + # Filter the walker based on the selected speaker IDs + selected_speaker_ids_ = set(selected_speaker_ids) if selected_speaker_ids is not None else None + if selected_speaker_ids_ is not None: + self._walker = [w for w in self._walker if int(w.split("_")[0]) in selected_speaker_ids_] + + # Filter the walker based on the maximum audio length + if max_audio_length is not None or min_audio_length > 0.0: + params = ( + self._path, + self._ext_audio, + self._ext_original_txt, + self._ext_normalized_txt, + min_audio_length, + max_audio_length, + ) + with Pool(cpu_count()) as p: + self._walker = [ + fileid + for fileid in p.map( + check_audio_length, + [(fileid, *params) for fileid in self._walker], + ) + if fileid is not None + ] + + def __getitem__(self, n: int) -> Tuple[Tensor, int, str, str, int, int, str]: + """Load the n-th sample from the dataset. + + Args: + n (int): The index of the sample to be loaded + + Returns: + Tuple of the following items; + + Tensor: + Waveform + int: + Sample rate + str: + Original text + str: + Normalized text + int: + Speaker ID + int: + Chapter ID + str: + Utterance ID + """ + fileid = self._walker[n] + return load_libritts_item( + fileid, + self._path, + self._ext_audio, + self._ext_original_txt, + self._ext_normalized_txt, + ) + + def __len__(self) -> int: + return len(self._walker) diff --git a/training/datasets/preprocessed_dataset.py b/training/datasets/preprocessed_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..afe20bd0e79f343896b8d2eb9d0ff91d3be34563 --- /dev/null +++ b/training/datasets/preprocessed_dataset.py @@ -0,0 +1,38 @@ +from typing import Any + +from torch.utils.data import Dataset + + +class PreprocessedDataset(Dataset): + r"""A PyTorch Dataset for holding preprocessed data. + + Attributes + data (list): The preprocessed data. + """ + + def __init__(self, data: Any): + r"""Initialize the PreprocessedDataset. + + Args: + data (list): The preprocessed data. + """ + self.data = data + + def __getitem__(self, index: int): + r"""Get the data at the given index. + + Args: + index (int): The index of the data to get. + + Returns: + The data at the given index. + """ + return self.data[index] + + def __len__(self): + r"""Get the number of data in the dataset. + + Returns + The number of data in the dataset. + """ + return len(self.data) diff --git a/training/datasets/speaker_id_mapping_libri.json b/training/datasets/speaker_id_mapping_libri.json new file mode 100644 index 0000000000000000000000000000000000000000..6612f37d4e6f853d9178780233123ef0cdf2f9be --- /dev/null +++ b/training/datasets/speaker_id_mapping_libri.json @@ -0,0 +1 @@ +{"14": 0, "16": 1, "17": 2, "19": 3, "20": 4, "22": 5, "23": 6, "25": 7, "26": 8, "27": 9, "28": 10, "29": 11, "30": 12, "31": 13, "32": 14, "36": 15, "37": 16, "38": 17, "39": 18, "40": 19, "44": 20, "45": 21, "46": 22, "47": 23, "49": 24, "51": 25, "52": 26, "54": 27, "55": 28, "56": 29, "57": 30, "58": 31, "60": 32, "61": 33, "62": 34, "64": 35, "65": 36, "66": 37, "70": 38, "75": 39, "77": 40, "78": 41, "79": 42, "81": 43, "82": 44, "83": 45, "84": 46, "85": 47, "87": 48, "89": 49, "90": 50, "91": 51, "92": 52, "93": 53, "94": 54, "98": 55, "100": 56, "101": 57, "102": 58, "103": 59, "104": 60, "107": 61, "110": 62, "111": 63, "112": 64, "114": 65, "115": 66, "116": 67, "118": 68, "119": 69, "121": 70, "122": 71, "123": 72, "125": 73, "126": 74, "127": 75, "128": 76, "133": 77, "147": 78, "149": 79, "150": 80, "151": 81, "152": 82, "153": 83, "154": 84, "157": 85, "159": 86, "161": 87, "163": 88, "166": 89, "167": 90, "168": 91, "173": 92, "174": 93, "175": 94, "176": 95, "177": 96, "188": 97, "192": 98, "196": 99, "198": 100, "199": 101, "200": 102, "201": 103, "202": 104, "203": 105, "204": 106, "205": 107, "207": 108, "208": 109, "209": 110, "210": 111, "211": 112, "215": 113, "216": 114, "217": 115, "218": 116, "224": 117, "225": 118, "226": 119, "227": 120, "228": 121, "229": 122, "231": 123, "233": 124, "237": 125, "238": 126, "240": 127, "242": 128, "243": 129, "245": 130, "246": 131, "248": 132, "249": 133, "250": 134, "251": 135, "252": 136, "253": 137, "254": 138, "255": 139, "258": 140, "260": 141, "263": 142, "264": 143, "265": 144, "272": 145, "273": 146, "274": 147, "277": 148, "278": 149, "283": 150, "288": 151, "289": 152, "294": 153, "296": 154, "298": 155, "302": 156, "303": 157, "307": 158, "310": 159, "311": 160, "313": 161, "317": 162, "318": 163, "319": 164, "322": 165, "323": 166, "328": 167, "329": 168, "331": 169, "332": 170, "335": 171, "336": 172, "337": 173, "339": 174, "340": 175, "345": 176, "348": 177, "353": 178, "359": 179, "362": 180, "365": 181, "366": 182, "367": 183, "369": 184, "373": 185, "374": 186, "377": 187, "380": 188, "392": 189, "398": 190, "402": 191, "403": 192, "404": 193, "405": 194, "408": 195, "409": 196, "412": 197, "413": 198, "421": 199, "422": 200, "426": 201, "428": 202, "432": 203, "434": 204, "439": 205, "441": 206, "444": 207, "445": 208, "446": 209, "448": 210, "451": 211, "453": 212, "454": 213, "458": 214, "459": 215, "460": 216, "464": 217, "466": 218, "470": 219, "472": 220, "474": 221, "475": 222, "476": 223, "479": 224, "480": 225, "481": 226, "483": 227, "487": 228, "489": 229, "492": 230, "497": 231, "500": 232, "501": 233, "505": 234, "510": 235, "511": 236, "512": 237, "517": 238, "525": 239, "533": 240, "534": 241, "542": 242, "543": 243, "544": 244, "548": 245, "549": 246, "551": 247, "557": 248, "559": 249, "561": 250, "567": 251, "568": 252, "569": 253, "572": 254, "576": 255, "580": 256, "581": 257, "583": 258, "584": 259, "585": 260, "587": 261, "589": 262, "593": 263, "594": 264, "596": 265, "597": 266, "598": 267, "606": 268, "608": 269, "612": 270, "613": 271, "614": 272, "622": 273, "625": 274, "636": 275, "637": 276, "639": 277, "644": 278, "652": 279, "663": 280, "664": 281, "666": 282, "667": 283, "669": 284, "671": 285, "672": 286, "679": 287, "681": 288, "684": 289, "688": 290, "690": 291, "696": 292, "698": 293, "699": 294, "700": 295, "705": 296, "707": 297, "708": 298, "711": 299, "712": 300, "713": 301, "716": 302, "718": 303, "720": 304, "724": 305, "726": 306, "727": 307, "728": 308, "730": 309, "731": 310, "737": 311, "742": 312, "753": 313, "764": 314, "766": 315, "770": 316, "774": 317, "777": 318, "778": 319, "779": 320, "780": 321, "781": 322, "782": 323, "783": 324, "789": 325, "791": 326, "792": 327, "797": 328, "803": 329, "806": 330, "807": 331, "810": 332, "811": 333, "815": 334, "816": 335, "820": 336, "826": 337, "829": 338, "830": 339, "831": 340, "834": 341, "835": 342, "836": 343, "839": 344, "844": 345, "845": 346, "846": 347, "850": 348, "851": 349, "868": 350, "876": 351, "882": 352, "884": 353, "886": 354, "887": 355, "895": 356, "899": 357, "908": 358, "909": 359, "911": 360, "915": 361, "920": 362, "921": 363, "922": 364, "923": 365, "925": 366, "927": 367, "937": 368, "948": 369, "949": 370, "951": 371, "953": 372, "954": 373, "956": 374, "957": 375, "960": 376, "964": 377, "968": 378, "969": 379, "976": 380, "978": 381, "979": 382, "982": 383, "984": 384, "985": 385, "986": 386, "1001": 387, "1006": 388, "1012": 389, "1018": 390, "1025": 391, "1027": 392, "1028": 393, "1031": 394, "1034": 395, "1040": 396, "1046": 397, "1049": 398, "1050": 399, "1051": 400, "1052": 401, "1053": 402, "1054": 403, "1058": 404, "1060": 405, "1061": 406, "1065": 407, "1066": 408, "1069": 409, "1079": 410, "1081": 411, "1084": 412, "1085": 413, "1088": 414, "1089": 415, "1092": 416, "1093": 417, "1094": 418, "1096": 419, "1097": 420, "1098": 421, "1100": 422, "1107": 423, "1110": 424, "1112": 425, "1116": 426, "1121": 427, "1124": 428, "1132": 429, "1152": 430, "1154": 431, "1160": 432, "1161": 433, "1165": 434, "1166": 435, "1168": 436, "1171": 437, "1175": 438, "1179": 439, "1182": 440, "1183": 441, "1184": 442, "1187": 443, "1188": 444, "1195": 445, "1200": 446, "1212": 447, "1221": 448, "1222": 449, "1224": 450, "1225": 451, "1226": 452, "1230": 453, "1235": 454, "1239": 455, "1241": 456, "1246": 457, "1250": 458, "1252": 459, "1255": 460, "1258": 461, "1259": 462, "1260": 463, "1261": 464, "1263": 465, "1264": 466, "1265": 467, "1266": 468, "1271": 469, "1272": 470, "1274": 471, "1280": 472, "1283": 473, "1284": 474, "1289": 475, "1290": 476, "1291": 477, "1296": 478, "1298": 479, "1311": 480, "1313": 481, "1316": 482, "1320": 483, "1322": 484, "1323": 485, "1331": 486, "1334": 487, "1335": 488, "1336": 489, "1337": 490, "1341": 491, "1342": 492, "1343": 493, "1347": 494, "1348": 495, "1349": 496, "1353": 497, "1355": 498, "1363": 499, "1365": 500, "1367": 501, "1370": 502, "1373": 503, "1374": 504, "1379": 505, "1382": 506, "1383": 507, "1384": 508, "1387": 509, "1390": 510, "1392": 511, "1401": 512, "1403": 513, "1413": 514, "1414": 515, "1417": 516, "1421": 517, "1422": 518, "1425": 519, "1430": 520, "1444": 521, "1445": 522, "1446": 523, "1447": 524, "1448": 525, "1455": 526, "1456": 527, "1460": 528, "1462": 529, "1463": 530, "1469": 531, "1472": 532, "1473": 533, "1474": 534, "1482": 535, "1485": 536, "1487": 537, "1492": 538, "1494": 539, "1495": 540, "1498": 541, "1502": 542, "1505": 543, "1509": 544, "1513": 545, "1535": 546, "1536": 547, "1544": 548, "1545": 549, "1547": 550, "1552": 551, "1553": 552, "1556": 553, "1559": 554, "1563": 555, "1564": 556, "1566": 557, "1569": 558, "1571": 559, "1572": 560, "1578": 561, "1579": 562, "1580": 563, "1585": 564, "1593": 565, "1594": 566, "1595": 567, "1601": 568, "1603": 569, "1607": 570, "1614": 571, "1618": 572, "1621": 573, "1624": 574, "1629": 575, "1630": 576, "1633": 577, "1634": 578, "1636": 579, "1638": 580, "1639": 581, "1641": 582, "1643": 583, "1645": 584, "1646": 585, "1647": 586, "1648": 587, "1649": 588, "1650": 589, "1651": 590, "1653": 591, "1664": 592, "1665": 593, "1668": 594, "1673": 595, "1674": 596, "1678": 597, "1679": 598, "1680": 599, "1681": 600, "1685": 601, "1686": 602, "1688": 603, "1690": 604, "1691": 605, "1693": 606, "1695": 607, "1696": 608, "1699": 609, "1701": 610, "1704": 611, "1705": 612, "1708": 613, "1710": 614, "1714": 615, "1715": 616, "1717": 617, "1721": 618, "1723": 619, "1724": 620, "1726": 621, "1731": 622, "1733": 623, "1734": 624, "1736": 625, "1737": 626, "1740": 627, "1743": 628, "1746": 629, "1748": 630, "1750": 631, "1752": 632, "1754": 633, "1756": 634, "1757": 635, "1760": 636, "1765": 637, "1767": 638, "1769": 639, "1772": 640, "1773": 641, "1776": 642, "1777": 643, "1779": 644, "1780": 645, "1784": 646, "1789": 647, "1795": 648, "1800": 649, "1801": 650, "1804": 651, "1806": 652, "1809": 653, "1811": 654, "1813": 655, "1815": 656, "1819": 657, "1825": 658, "1826": 659, "1827": 660, "1828": 661, "1841": 662, "1844": 663, "1845": 664, "1846": 665, "1849": 666, "1851": 667, "1859": 668, "1863": 669, "1867": 670, "1868": 671, "1870": 672, "1874": 673, "1878": 674, "1885": 675, "1898": 676, "1901": 677, "1903": 678, "1913": 679, "1914": 680, "1919": 681, "1920": 682, "1923": 683, "1924": 684, "1926": 685, "1931": 686, "1933": 687, "1938": 688, "1943": 689, "1944": 690, "1958": 691, "1961": 692, "1963": 693, "1968": 694, "1970": 695, "1974": 696, "1977": 697, "1985": 698, "1987": 699, "1988": 700, "1989": 701, "1992": 702, "1993": 703, "1995": 704, "1998": 705, "2001": 706, "2002": 707, "2003": 708, "2004": 709, "2007": 710, "2010": 711, "2012": 712, "2013": 713, "2021": 714, "2026": 715, "2033": 716, "2035": 717, "2039": 718, "2042": 719, "2045": 720, "2046": 721, "2050": 722, "2051": 723, "2053": 724, "2056": 725, "2060": 726, "2061": 727, "2062": 728, "2063": 729, "2067": 730, "2068": 731, "2074": 732, "2078": 733, "2085": 734, "2086": 735, "2089": 736, "2090": 737, "2092": 738, "2093": 739, "2094": 740, "2096": 741, "2100": 742, "2104": 743, "2110": 744, "2113": 745, "2122": 746, "2127": 747, "2133": 748, "2136": 749, "2137": 750, "2140": 751, "2143": 752, "2146": 753, "2148": 754, "2149": 755, "2152": 756, "2156": 757, "2159": 758, "2162": 759, "2167": 760, "2182": 761, "2185": 762, "2194": 763, "2195": 764, "2196": 765, "2198": 766, "2201": 767, "2204": 768, "2208": 769, "2229": 770, "2230": 771, "2234": 772, "2237": 773, "2238": 774, "2240": 775, "2246": 776, "2254": 777, "2256": 778, "2262": 779, "2269": 780, "2270": 781, "2272": 782, "2273": 783, "2275": 784, "2276": 785, "2277": 786, "2279": 787, "2284": 788, "2285": 789, "2288": 790, "2289": 791, "2292": 792, "2294": 793, "2297": 794, "2299": 795, "2300": 796, "2301": 797, "2309": 798, "2312": 799, "2319": 800, "2334": 801, "2339": 802, "2341": 803, "2346": 804, "2348": 805, "2351": 806, "2356": 807, "2361": 808, "2364": 809, "2368": 810, "2374": 811, "2380": 812, "2384": 813, "2388": 814, "2391": 815, "2393": 816, "2397": 817, "2401": 818, "2404": 819, "2405": 820, "2407": 821, "2411": 822, "2412": 823, "2414": 824, "2416": 825, "2427": 826, "2428": 827, "2436": 828, "2437": 829, "2445": 830, "2448": 831, "2473": 832, "2481": 833, "2485": 834, "2487": 835, "2488": 836, "2491": 837, "2494": 838, "2496": 839, "2498": 840, "2499": 841, "2504": 842, "2506": 843, "2512": 844, "2514": 845, "2517": 846, "2518": 847, "2522": 848, "2526": 849, "2531": 850, "2532": 851, "2533": 852, "2541": 853, "2544": 854, "2545": 855, "2552": 856, "2553": 857, "2562": 858, "2568": 859, "2570": 860, "2573": 861, "2574": 862, "2577": 863, "2581": 864, "2582": 865, "2587": 866, "2588": 867, "2589": 868, "2592": 869, "2598": 870, "2606": 871, "2607": 872, "2609": 873, "2618": 874, "2624": 875, "2625": 876, "2628": 877, "2638": 878, "2652": 879, "2654": 880, "2660": 881, "2671": 882, "2673": 883, "2674": 884, "2676": 885, "2688": 886, "2691": 887, "2694": 888, "2696": 889, "2709": 890, "2712": 891, "2724": 892, "2730": 893, "2733": 894, "2735": 895, "2740": 896, "2741": 897, "2748": 898, "2751": 899, "2754": 900, "2758": 901, "2762": 902, "2764": 903, "2769": 904, "2774": 905, "2775": 906, "2785": 907, "2787": 908, "2790": 909, "2792": 910, "2803": 911, "2812": 912, "2815": 913, "2816": 914, "2817": 915, "2823": 916, "2825": 917, "2827": 918, "2830": 919, "2834": 920, "2836": 921, "2843": 922, "2853": 923, "2854": 924, "2882": 925, "2893": 926, "2895": 927, "2902": 928, "2909": 929, "2910": 930, "2911": 931, "2919": 932, "2920": 933, "2925": 934, "2929": 935, "2930": 936, "2943": 937, "2946": 938, "2952": 939, "2960": 940, "2961": 941, "2967": 942, "2971": 943, "2975": 944, "2979": 945, "2985": 946, "2988": 947, "2989": 948, "2990": 949, "2992": 950, "2997": 951, "2998": 952, "2999": 953, "3000": 954, "3001": 955, "3003": 956, "3005": 957, "3006": 958, "3008": 959, "3009": 960, "3020": 961, "3021": 962, "3025": 963, "3032": 964, "3033": 965, "3045": 966, "3046": 967, "3053": 968, "3054": 969, "3060": 970, "3063": 971, "3070": 972, "3072": 973, "3079": 974, "3080": 975, "3081": 976, "3082": 977, "3083": 978, "3088": 979, "3090": 980, "3092": 981, "3094": 982, "3097": 983, "3098": 984, "3100": 985, "3105": 986, "3109": 987, "3112": 988, "3114": 989, "3118": 990, "3119": 991, "3125": 992, "3132": 993, "3135": 994, "3137": 995, "3138": 996, "3142": 997, "3143": 998, "3144": 999, "3148": 1000, "3157": 1001, "3168": 1002, "3170": 1003, "3171": 1004, "3172": 1005, "3179": 1006, "3180": 1007, "3185": 1008, "3187": 1009, "3192": 1010, "3196": 1011, "3214": 1012, "3215": 1013, "3221": 1014, "3224": 1015, "3227": 1016, "3228": 1017, "3230": 1018, "3235": 1019, "3238": 1020, "3240": 1021, "3242": 1022, "3244": 1023, "3245": 1024, "3257": 1025, "3258": 1026, "3259": 1027, "3261": 1028, "3268": 1029, "3271": 1030, "3272": 1031, "3274": 1032, "3285": 1033, "3288": 1034, "3289": 1035, "3290": 1036, "3294": 1037, "3307": 1038, "3314": 1039, "3318": 1040, "3319": 1041, "3328": 1042, "3330": 1043, "3331": 1044, "3334": 1045, "3340": 1046, "3346": 1047, "3347": 1048, "3356": 1049, "3357": 1050, "3361": 1051, "3368": 1052, "3370": 1053, "3373": 1054, "3374": 1055, "3379": 1056, "3380": 1057, "3381": 1058, "3389": 1059, "3394": 1060, "3400": 1061, "3409": 1062, "3411": 1063, "3417": 1064, "3433": 1065, "3436": 1066, "3440": 1067, "3446": 1068, "3448": 1069, "3465": 1070, "3467": 1071, "3470": 1072, "3479": 1073, "3482": 1074, "3483": 1075, "3486": 1076, "3488": 1077, "3490": 1078, "3493": 1079, "3500": 1080, "3503": 1081, "3513": 1082, "3521": 1083, "3526": 1084, "3528": 1085, "3536": 1086, "3537": 1087, "3538": 1088, "3540": 1089, "3541": 1090, "3546": 1091, "3547": 1092, "3549": 1093, "3551": 1094, "3553": 1095, "3554": 1096, "3557": 1097, "3559": 1098, "3564": 1099, "3567": 1100, "3570": 1101, "3571": 1102, "3575": 1103, "3576": 1104, "3584": 1105, "3587": 1106, "3588": 1107, "3592": 1108, "3595": 1109, "3598": 1110, "3606": 1111, "3607": 1112, "3615": 1113, "3618": 1114, "3630": 1115, "3638": 1116, "3641": 1117, "3645": 1118, "3647": 1119, "3650": 1120, "3654": 1121, "3656": 1122, "3657": 1123, "3660": 1124, "3663": 1125, "3664": 1126, "3665": 1127, "3675": 1128, "3679": 1129, "3681": 1130, "3686": 1131, "3691": 1132, "3698": 1133, "3699": 1134, "3703": 1135, "3717": 1136, "3723": 1137, "3728": 1138, "3729": 1139, "3733": 1140, "3738": 1141, "3744": 1142, "3747": 1143, "3752": 1144, "3757": 1145, "3764": 1146, "3779": 1147, "3780": 1148, "3781": 1149, "3783": 1150, "3790": 1151, "3792": 1152, "3793": 1153, "3796": 1154, "3798": 1155, "3807": 1156, "3816": 1157, "3819": 1158, "3825": 1159, "3830": 1160, "3835": 1161, "3843": 1162, "3845": 1163, "3848": 1164, "3851": 1165, "3852": 1166, "3853": 1167, "3857": 1168, "3864": 1169, "3866": 1170, "3867": 1171, "3869": 1172, "3871": 1173, "3876": 1174, "3879": 1175, "3885": 1176, "3889": 1177, "3894": 1178, "3895": 1179, "3896": 1180, "3905": 1181, "3906": 1182, "3909": 1183, "3911": 1184, "3912": 1185, "3914": 1186, "3915": 1187, "3922": 1188, "3923": 1189, "3925": 1190, "3926": 1191, "3927": 1192, "3928": 1193, "3934": 1194, "3945": 1195, "3947": 1196, "3955": 1197, "3959": 1198, "3962": 1199, "3967": 1200, "3969": 1201, "3972": 1202, "3977": 1203, "3979": 1204, "3982": 1205, "3983": 1206, "3989": 1207, "3990": 1208, "3992": 1209, "3994": 1210, "3997": 1211, "4005": 1212, "4009": 1213, "4010": 1214, "4013": 1215, "4014": 1216, "4015": 1217, "4017": 1218, "4018": 1219, "4019": 1220, "4020": 1221, "4021": 1222, "4034": 1223, "4039": 1224, "4042": 1225, "4044": 1226, "4051": 1227, "4054": 1228, "4057": 1229, "4059": 1230, "4063": 1231, "4064": 1232, "4071": 1233, "4077": 1234, "4078": 1235, "4085": 1236, "4088": 1237, "4090": 1238, "4098": 1239, "4104": 1240, "4108": 1241, "4110": 1242, "4111": 1243, "4116": 1244, "4122": 1245, "4133": 1246, "4137": 1247, "4138": 1248, "4145": 1249, "4148": 1250, "4152": 1251, "4153": 1252, "4156": 1253, "4160": 1254, "4161": 1255, "4172": 1256, "4174": 1257, "4179": 1258, "4189": 1259, "4191": 1260, "4192": 1261, "4193": 1262, "4195": 1263, "4196": 1264, "4198": 1265, "4205": 1266, "4211": 1267, "4214": 1268, "4216": 1269, "4217": 1270, "4218": 1271, "4222": 1272, "4225": 1273, "4226": 1274, "4234": 1275, "4235": 1276, "4236": 1277, "4238": 1278, "4243": 1279, "4246": 1280, "4257": 1281, "4260": 1282, "4262": 1283, "4263": 1284, "4267": 1285, "4273": 1286, "4277": 1287, "4278": 1288, "4280": 1289, "4289": 1290, "4290": 1291, "4294": 1292, "4295": 1293, "4297": 1294, "4305": 1295, "4310": 1296, "4313": 1297, "4321": 1298, "4323": 1299, "4327": 1300, "4331": 1301, "4335": 1302, "4340": 1303, "4344": 1304, "4345": 1305, "4350": 1306, "4352": 1307, "4356": 1308, "4358": 1309, "4362": 1310, "4363": 1311, "4379": 1312, "4381": 1313, "4396": 1314, "4397": 1315, "4402": 1316, "4406": 1317, "4407": 1318, "4411": 1319, "4415": 1320, "4420": 1321, "4422": 1322, "4423": 1323, "4425": 1324, "4427": 1325, "4428": 1326, "4433": 1327, "4434": 1328, "4438": 1329, "4441": 1330, "4442": 1331, "4443": 1332, "4446": 1333, "4447": 1334, "4455": 1335, "4463": 1336, "4474": 1337, "4480": 1338, "4481": 1339, "4484": 1340, "4487": 1341, "4490": 1342, "4492": 1343, "4495": 1344, "4507": 1345, "4511": 1346, "4513": 1347, "4515": 1348, "4519": 1349, "4520": 1350, "4535": 1351, "4545": 1352, "4546": 1353, "4549": 1354, "4563": 1355, "4570": 1356, "4572": 1357, "4576": 1358, "4583": 1359, "4586": 1360, "4590": 1361, "4591": 1362, "4592": 1363, "4594": 1364, "4595": 1365, "4598": 1366, "4599": 1367, "4629": 1368, "4640": 1369, "4652": 1370, "4659": 1371, "4660": 1372, "4667": 1373, "4680": 1374, "4681": 1375, "4687": 1376, "4693": 1377, "4697": 1378, "4699": 1379, "4701": 1380, "4703": 1381, "4705": 1382, "4706": 1383, "4710": 1384, "4712": 1385, "4719": 1386, "4731": 1387, "4733": 1388, "4734": 1389, "4738": 1390, "4741": 1391, "4742": 1392, "4744": 1393, "4748": 1394, "4750": 1395, "4757": 1396, "4766": 1397, "4767": 1398, "4770": 1399, "4771": 1400, "4773": 1401, "4779": 1402, "4788": 1403, "4791": 1404, "4799": 1405, "4800": 1406, "4806": 1407, "4807": 1408, "4813": 1409, "4821": 1410, "4824": 1411, "4830": 1412, "4831": 1413, "4836": 1414, "4837": 1415, "4839": 1416, "4841": 1417, "4846": 1418, "4848": 1419, "4852": 1420, "4853": 1421, "4854": 1422, "4856": 1423, "4859": 1424, "4860": 1425, "4863": 1426, "4872": 1427, "4894": 1428, "4898": 1429, "4899": 1430, "4910": 1431, "4915": 1432, "4926": 1433, "4930": 1434, "4931": 1435, "4936": 1436, "4945": 1437, "4948": 1438, "4955": 1439, "4957": 1440, "4958": 1441, "4959": 1442, "4964": 1443, "4965": 1444, "4967": 1445, "4969": 1446, "4970": 1447, "4973": 1448, "4979": 1449, "4991": 1450, "4992": 1451, "4993": 1452, "5000": 1453, "5002": 1454, "5005": 1455, "5007": 1456, "5009": 1457, "5012": 1458, "5013": 1459, "5019": 1460, "5022": 1461, "5023": 1462, "5029": 1463, "5036": 1464, "5038": 1465, "5039": 1466, "5043": 1467, "5044": 1468, "5045": 1469, "5049": 1470, "5054": 1471, "5060": 1472, "5062": 1473, "5063": 1474, "5076": 1475, "5077": 1476, "5082": 1477, "5092": 1478, "5093": 1479, "5101": 1480, "5104": 1481, "5105": 1482, "5115": 1483, "5118": 1484, "5123": 1485, "5126": 1486, "5132": 1487, "5133": 1488, "5136": 1489, "5139": 1490, "5141": 1491, "5142": 1492, "5147": 1493, "5152": 1494, "5154": 1495, "5157": 1496, "5163": 1497, "5164": 1498, "5172": 1499, "5181": 1500, "5183": 1501, "5185": 1502, "5186": 1503, "5189": 1504, "5190": 1505, "5192": 1506, "5198": 1507, "5199": 1508, "5206": 1509, "5217": 1510, "5220": 1511, "5224": 1512, "5230": 1513, "5233": 1514, "5239": 1515, "5242": 1516, "5244": 1517, "5245": 1518, "5246": 1519, "5248": 1520, "5252": 1521, "5261": 1522, "5266": 1523, "5269": 1524, "5271": 1525, "5278": 1526, "5280": 1527, "5285": 1528, "5287": 1529, "5290": 1530, "5293": 1531, "5296": 1532, "5299": 1533, "5304": 1534, "5319": 1535, "5321": 1536, "5322": 1537, "5325": 1538, "5328": 1539, "5333": 1540, "5337": 1541, "5338": 1542, "5339": 1543, "5340": 1544, "5350": 1545, "5355": 1546, "5361": 1547, "5375": 1548, "5379": 1549, "5386": 1550, "5389": 1551, "5390": 1552, "5393": 1553, "5400": 1554, "5401": 1555, "5405": 1556, "5412": 1557, "5424": 1558, "5429": 1559, "5439": 1560, "5442": 1561, "5445": 1562, "5448": 1563, "5456": 1564, "5459": 1565, "5460": 1566, "5463": 1567, "5468": 1568, "5471": 1569, "5480": 1570, "5484": 1571, "5487": 1572, "5489": 1573, "5506": 1574, "5513": 1575, "5514": 1576, "5519": 1577, "5536": 1578, "5538": 1579, "5543": 1580, "5545": 1581, "5561": 1582, "5565": 1583, "5567": 1584, "5569": 1585, "5570": 1586, "5583": 1587, "5588": 1588, "5604": 1589, "5606": 1590, "5618": 1591, "5620": 1592, "5622": 1593, "5628": 1594, "5635": 1595, "5636": 1596, "5637": 1597, "5639": 1598, "5641": 1599, "5649": 1600, "5652": 1601, "5653": 1602, "5655": 1603, "5656": 1604, "5660": 1605, "5661": 1606, "5665": 1607, "5671": 1608, "5672": 1609, "5678": 1610, "5682": 1611, "5683": 1612, "5684": 1613, "5688": 1614, "5694": 1615, "5700": 1616, "5703": 1617, "5712": 1618, "5717": 1619, "5719": 1620, "5720": 1621, "5723": 1622, "5724": 1623, "5725": 1624, "5727": 1625, "5731": 1626, "5733": 1627, "5735": 1628, "5740": 1629, "5746": 1630, "5750": 1631, "5756": 1632, "5764": 1633, "5765": 1634, "5767": 1635, "5772": 1636, "5776": 1637, "5778": 1638, "5781": 1639, "5784": 1640, "5789": 1641, "5791": 1642, "5796": 1643, "5802": 1644, "5808": 1645, "5809": 1646, "5810": 1647, "5825": 1648, "5826": 1649, "5831": 1650, "5837": 1651, "5840": 1652, "5849": 1653, "5854": 1654, "5860": 1655, "5867": 1656, "5868": 1657, "5874": 1658, "5876": 1659, "5883": 1660, "5886": 1661, "5890": 1662, "5893": 1663, "5894": 1664, "5895": 1665, "5906": 1666, "5909": 1667, "5910": 1668, "5911": 1669, "5913": 1670, "5914": 1671, "5918": 1672, "5929": 1673, "5933": 1674, "5935": 1675, "5940": 1676, "5949": 1677, "5951": 1678, "5952": 1679, "5968": 1680, "5970": 1681, "5975": 1682, "5977": 1683, "5979": 1684, "5980": 1685, "5983": 1686, "5984": 1687, "5985": 1688, "5993": 1689, "6000": 1690, "6003": 1691, "6006": 1692, "6009": 1693, "6010": 1694, "6014": 1695, "6019": 1696, "6025": 1697, "6030": 1698, "6032": 1699, "6035": 1700, "6037": 1701, "6038": 1702, "6051": 1703, "6054": 1704, "6060": 1705, "6064": 1706, "6065": 1707, "6070": 1708, "6072": 1709, "6075": 1710, "6076": 1711, "6077": 1712, "6078": 1713, "6080": 1714, "6081": 1715, "6082": 1716, "6084": 1717, "6087": 1718, "6088": 1719, "6097": 1720, "6098": 1721, "6099": 1722, "6102": 1723, "6104": 1724, "6106": 1725, "6111": 1726, "6115": 1727, "6119": 1728, "6120": 1729, "6121": 1730, "6123": 1731, "6126": 1732, "6127": 1733, "6128": 1734, "6131": 1735, "6135": 1736, "6138": 1737, "6139": 1738, "6145": 1739, "6147": 1740, "6153": 1741, "6157": 1742, "6159": 1743, "6160": 1744, "6167": 1745, "6173": 1746, "6177": 1747, "6178": 1748, "6181": 1749, "6184": 1750, "6188": 1751, "6189": 1752, "6196": 1753, "6199": 1754, "6206": 1755, "6209": 1756, "6211": 1757, "6215": 1758, "6221": 1759, "6224": 1760, "6227": 1761, "6232": 1762, "6233": 1763, "6235": 1764, "6236": 1765, "6241": 1766, "6242": 1767, "6248": 1768, "6249": 1769, "6251": 1770, "6254": 1771, "6258": 1772, "6267": 1773, "6269": 1774, "6272": 1775, "6276": 1776, "6281": 1777, "6284": 1778, "6286": 1779, "6288": 1780, "6294": 1781, "6295": 1782, "6300": 1783, "6308": 1784, "6311": 1785, "6313": 1786, "6317": 1787, "6319": 1788, "6323": 1789, "6324": 1790, "6330": 1791, "6332": 1792, "6333": 1793, "6339": 1794, "6341": 1795, "6345": 1796, "6346": 1797, "6351": 1798, "6352": 1799, "6353": 1800, "6356": 1801, "6358": 1802, "6359": 1803, "6364": 1804, "6367": 1805, "6368": 1806, "6370": 1807, "6371": 1808, "6373": 1809, "6377": 1810, "6378": 1811, "6385": 1812, "6388": 1813, "6391": 1814, "6395": 1815, "6399": 1816, "6402": 1817, "6406": 1818, "6407": 1819, "6411": 1820, "6415": 1821, "6418": 1822, "6426": 1823, "6432": 1824, "6436": 1825, "6437": 1826, "6446": 1827, "6454": 1828, "6455": 1829, "6458": 1830, "6459": 1831, "6467": 1832, "6476": 1833, "6482": 1834, "6484": 1835, "6488": 1836, "6492": 1837, "6494": 1838, "6497": 1839, "6499": 1840, "6505": 1841, "6506": 1842, "6509": 1843, "6510": 1844, "6512": 1845, "6513": 1846, "6518": 1847, "6519": 1848, "6529": 1849, "6531": 1850, "6533": 1851, "6534": 1852, "6535": 1853, "6538": 1854, "6539": 1855, "6540": 1856, "6544": 1857, "6548": 1858, "6549": 1859, "6550": 1860, "6553": 1861, "6555": 1862, "6557": 1863, "6563": 1864, "6567": 1865, "6568": 1866, "6574": 1867, "6575": 1868, "6583": 1869, "6590": 1870, "6594": 1871, "6599": 1872, "6609": 1873, "6610": 1874, "6614": 1875, "6620": 1876, "6625": 1877, "6627": 1878, "6636": 1879, "6637": 1880, "6641": 1881, "6643": 1882, "6652": 1883, "6660": 1884, "6668": 1885, "6670": 1886, "6673": 1887, "6674": 1888, "6676": 1889, "6683": 1890, "6685": 1891, "6686": 1892, "6687": 1893, "6689": 1894, "6690": 1895, "6694": 1896, "6695": 1897, "6696": 1898, "6701": 1899, "6705": 1900, "6707": 1901, "6709": 1902, "6713": 1903, "6724": 1904, "6726": 1905, "6727": 1906, "6733": 1907, "6735": 1908, "6741": 1909, "6743": 1910, "6746": 1911, "6747": 1912, "6749": 1913, "6752": 1914, "6753": 1915, "6754": 1916, "6758": 1917, "6763": 1918, "6773": 1919, "6777": 1920, "6782": 1921, "6784": 1922, "6788": 1923, "6792": 1924, "6794": 1925, "6798": 1926, "6804": 1927, "6807": 1928, "6818": 1929, "6821": 1930, "6828": 1931, "6829": 1932, "6836": 1933, "6841": 1934, "6846": 1935, "6848": 1936, "6849": 1937, "6853": 1938, "6865": 1939, "6875": 1940, "6877": 1941, "6880": 1942, "6882": 1943, "6883": 1944, "6892": 1945, "6895": 1946, "6902": 1947, "6904": 1948, "6906": 1949, "6912": 1950, "6913": 1951, "6914": 1952, "6918": 1953, "6923": 1954, "6924": 1955, "6925": 1956, "6927": 1957, "6930": 1958, "6937": 1959, "6938": 1960, "6943": 1961, "6945": 1962, "6947": 1963, "6950": 1964, "6951": 1965, "6954": 1966, "6956": 1967, "6962": 1968, "6963": 1969, "6965": 1970, "6967": 1971, "6974": 1972, "6978": 1973, "6981": 1974, "6993": 1975, "7000": 1976, "7001": 1977, "7008": 1978, "7009": 1979, "7011": 1980, "7012": 1981, "7018": 1982, "7021": 1983, "7026": 1984, "7030": 1985, "7046": 1986, "7051": 1987, "7055": 1988, "7059": 1989, "7061": 1990, "7062": 1991, "7065": 1992, "7067": 1993, "7069": 1994, "7073": 1995, "7078": 1996, "7079": 1997, "7085": 1998, "7090": 1999, "7092": 2000, "7095": 2001, "7096": 2002, "7097": 2003, "7105": 2004, "7107": 2005, "7113": 2006, "7117": 2007, "7120": 2008, "7121": 2009, "7125": 2010, "7126": 2011, "7127": 2012, "7128": 2013, "7131": 2014, "7134": 2015, "7135": 2016, "7138": 2017, "7139": 2018, "7140": 2019, "7143": 2020, "7145": 2021, "7147": 2022, "7148": 2023, "7150": 2024, "7155": 2025, "7169": 2026, "7170": 2027, "7176": 2028, "7177": 2029, "7178": 2030, "7188": 2031, "7189": 2032, "7190": 2033, "7197": 2034, "7198": 2035, "7199": 2036, "7205": 2037, "7208": 2038, "7215": 2039, "7218": 2040, "7220": 2041, "7223": 2042, "7226": 2043, "7228": 2044, "7229": 2045, "7238": 2046, "7239": 2047, "7240": 2048, "7241": 2049, "7242": 2050, "7245": 2051, "7246": 2052, "7247": 2053, "7250": 2054, "7255": 2055, "7258": 2056, "7263": 2057, "7264": 2058, "7265": 2059, "7276": 2060, "7277": 2061, "7278": 2062, "7285": 2063, "7286": 2064, "7294": 2065, "7297": 2066, "7299": 2067, "7301": 2068, "7302": 2069, "7307": 2070, "7312": 2071, "7313": 2072, "7314": 2073, "7315": 2074, "7316": 2075, "7318": 2076, "7320": 2077, "7326": 2078, "7327": 2079, "7331": 2080, "7333": 2081, "7335": 2082, "7337": 2083, "7338": 2084, "7339": 2085, "7342": 2086, "7346": 2087, "7348": 2088, "7354": 2089, "7357": 2090, "7360": 2091, "7367": 2092, "7376": 2093, "7383": 2094, "7384": 2095, "7387": 2096, "7389": 2097, "7391": 2098, "7392": 2099, "7395": 2100, "7398": 2101, "7402": 2102, "7408": 2103, "7416": 2104, "7423": 2105, "7424": 2106, "7433": 2107, "7434": 2108, "7436": 2109, "7437": 2110, "7445": 2111, "7447": 2112, "7448": 2113, "7460": 2114, "7463": 2115, "7467": 2116, "7475": 2117, "7478": 2118, "7480": 2119, "7481": 2120, "7484": 2121, "7491": 2122, "7492": 2123, "7495": 2124, "7498": 2125, "7502": 2126, "7505": 2127, "7507": 2128, "7510": 2129, "7511": 2130, "7512": 2131, "7514": 2132, "7515": 2133, "7517": 2134, "7518": 2135, "7520": 2136, "7522": 2137, "7525": 2138, "7538": 2139, "7540": 2140, "7552": 2141, "7553": 2142, "7555": 2143, "7556": 2144, "7558": 2145, "7559": 2146, "7561": 2147, "7565": 2148, "7569": 2149, "7584": 2150, "7585": 2151, "7594": 2152, "7597": 2153, "7601": 2154, "7603": 2155, "7607": 2156, "7608": 2157, "7609": 2158, "7618": 2159, "7635": 2160, "7640": 2161, "7641": 2162, "7644": 2163, "7647": 2164, "7649": 2165, "7654": 2166, "7657": 2167, "7665": 2168, "7672": 2169, "7679": 2170, "7683": 2171, "7687": 2172, "7688": 2173, "7691": 2174, "7697": 2175, "7699": 2176, "7700": 2177, "7702": 2178, "7704": 2179, "7705": 2180, "7708": 2181, "7713": 2182, "7717": 2183, "7720": 2184, "7729": 2185, "7730": 2186, "7732": 2187, "7733": 2188, "7737": 2189, "7739": 2190, "7746": 2191, "7749": 2192, "7752": 2193, "7754": 2194, "7756": 2195, "7762": 2196, "7764": 2197, "7766": 2198, "7769": 2199, "7777": 2200, "7780": 2201, "7783": 2202, "7786": 2203, "7789": 2204, "7794": 2205, "7795": 2206, "7796": 2207, "7800": 2208, "7802": 2209, "7809": 2210, "7816": 2211, "7823": 2212, "7825": 2213, "7826": 2214, "7828": 2215, "7832": 2216, "7833": 2217, "7835": 2218, "7837": 2219, "7839": 2220, "7843": 2221, "7848": 2222, "7850": 2223, "7859": 2224, "7867": 2225, "7868": 2226, "7871": 2227, "7874": 2228, "7879": 2229, "7881": 2230, "7883": 2231, "7886": 2232, "7892": 2233, "7898": 2234, "7902": 2235, "7909": 2236, "7910": 2237, "7912": 2238, "7923": 2239, "7925": 2240, "7926": 2241, "7932": 2242, "7933": 2243, "7938": 2244, "7939": 2245, "7942": 2246, "7945": 2247, "7946": 2248, "7949": 2249, "7956": 2250, "7957": 2251, "7959": 2252, "7962": 2253, "7967": 2254, "7975": 2255, "7976": 2256, "7981": 2257, "7982": 2258, "7988": 2259, "7991": 2260, "7994": 2261, "7995": 2262, "7997": 2263, "8005": 2264, "8006": 2265, "8008": 2266, "8009": 2267, "8011": 2268, "8012": 2269, "8014": 2270, "8015": 2271, "8023": 2272, "8028": 2273, "8033": 2274, "8040": 2275, "8042": 2276, "8044": 2277, "8050": 2278, "8051": 2279, "8057": 2280, "8058": 2281, "8063": 2282, "8066": 2283, "8071": 2284, "8072": 2285, "8075": 2286, "8080": 2287, "8087": 2288, "8088": 2289, "8095": 2290, "8097": 2291, "8098": 2292, "8108": 2293, "8112": 2294, "8113": 2295, "8118": 2296, "8119": 2297, "8123": 2298, "8131": 2299, "8138": 2300, "8142": 2301, "8143": 2302, "8148": 2303, "8152": 2304, "8156": 2305, "8163": 2306, "8164": 2307, "8168": 2308, "8169": 2309, "8172": 2310, "8173": 2311, "8176": 2312, "8180": 2313, "8183": 2314, "8188": 2315, "8190": 2316, "8193": 2317, "8194": 2318, "8195": 2319, "8197": 2320, "8199": 2321, "8200": 2322, "8208": 2323, "8215": 2324, "8222": 2325, "8224": 2326, "8225": 2327, "8226": 2328, "8228": 2329, "8230": 2330, "8238": 2331, "8240": 2332, "8242": 2333, "8245": 2334, "8246": 2335, "8250": 2336, "8254": 2337, "8259": 2338, "8262": 2339, "8266": 2340, "8272": 2341, "8273": 2342, "8280": 2343, "8288": 2344, "8291": 2345, "8295": 2346, "8296": 2347, "8297": 2348, "8300": 2349, "8302": 2350, "8307": 2351, "8312": 2352, "8316": 2353, "8321": 2354, "8322": 2355, "8324": 2356, "8328": 2357, "8329": 2358, "8334": 2359, "8337": 2360, "8346": 2361, "8347": 2362, "8356": 2363, "8367": 2364, "8382": 2365, "8388": 2366, "8389": 2367, "8392": 2368, "8394": 2369, "8396": 2370, "8401": 2371, "8404": 2372, "8410": 2373, "8413": 2374, "8414": 2375, "8415": 2376, "8419": 2377, "8421": 2378, "8422": 2379, "8424": 2380, "8425": 2381, "8430": 2382, "8432": 2383, "8441": 2384, "8443": 2385, "8444": 2386, "8445": 2387, "8447": 2388, "8455": 2389, "8459": 2390, "8461": 2391, "8463": 2392, "8464": 2393, "8465": 2394, "8466": 2395, "8468": 2396, "8470": 2397, "8474": 2398, "8476": 2399, "8479": 2400, "8490": 2401, "8494": 2402, "8498": 2403, "8499": 2404, "8500": 2405, "8506": 2406, "8527": 2407, "8531": 2408, "8534": 2409, "8536": 2410, "8543": 2411, "8544": 2412, "8545": 2413, "8555": 2414, "8565": 2415, "8573": 2416, "8575": 2417, "8576": 2418, "8580": 2419, "8587": 2420, "8590": 2421, "8591": 2422, "8592": 2423, "8605": 2424, "8609": 2425, "8619": 2426, "8625": 2427, "8629": 2428, "8630": 2429, "8631": 2430, "8632": 2431, "8635": 2432, "8643": 2433, "8644": 2434, "8664": 2435, "8666": 2436, "8671": 2437, "8675": 2438, "8677": 2439, "8678": 2440, "8684": 2441, "8687": 2442, "8699": 2443, "8705": 2444, "8710": 2445, "8713": 2446, "8718": 2447, "8722": 2448, "8725": 2449, "8742": 2450, "8747": 2451, "8753": 2452, "8758": 2453, "8765": 2454, "8770": 2455, "8771": 2456, "8772": 2457, "8776": 2458, "8778": 2459, "8786": 2460, "8791": 2461, "8797": 2462, "8799": 2463, "8803": 2464, "8808": 2465, "8820": 2466, "8824": 2467, "8825": 2468, "8838": 2469, "8842": 2470, "8846": 2471, "8848": 2472, "8855": 2473, "8867": 2474, "8875": 2475, "8879": 2476, "8887": 2477, "8897": 2478, "8975": 2479, "9000": 2480, "9022": 2481, "9023": 2482, "9026": 2483} \ No newline at end of file diff --git a/training/datasets/tests/__init__.py b/training/datasets/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/training/datasets/tests/test_hifi_gan_dataset.py b/training/datasets/tests/test_hifi_gan_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..05bdf11ce555a0de67012628749b1c790073aa07 --- /dev/null +++ b/training/datasets/tests/test_hifi_gan_dataset.py @@ -0,0 +1,46 @@ +from dataclasses import asdict +from pathlib import Path +import unittest + +from training.datasets import HifiGanDataset + + +class TestHifiGanDataset(unittest.TestCase): + def setUp(self): + self.cache_dir = "datasets_cache" + self.dataset = HifiGanDataset(cache_dir=self.cache_dir, cache=True) + + def test_len(self): + # Test that the length of the dataset is correct + self.assertIsInstance(len(self.dataset), int) + + def test_get_cache_subdir_path(self): + idx = 1234 + expected_path = Path(self.cache_dir) / "cache-hifigan-dataset" / "2000" + self.assertEqual(self.dataset.get_cache_subdir_path(idx), expected_path) + + def test_get_cache_file_path(self): + idx = 1234 + expected_path = ( + Path(self.cache_dir) / "cache-hifigan-dataset" / "2000" / f"{idx}.pt" + ) + self.assertEqual(self.dataset.get_cache_file_path(idx), expected_path) + + def test_getitem(self): + # Test that getting an item from the dataset returns a HifiGANItem + item = self.dataset[0] + self.assertIsInstance(item, tuple) + + def test_iter(self): + # Test that the dataset is iterable + for item in self.dataset: + self.assertIsInstance(item, tuple) + break + + def test_cache(self): + cache_file = self.dataset.get_cache_file_path(0) + self.assertTrue(cache_file.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/datasets/tests/test_hifi_libri_dataset.py b/training/datasets/tests/test_hifi_libri_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..e38f4c0f753fea56aae920fa70b31b6bbcf6fd3d --- /dev/null +++ b/training/datasets/tests/test_hifi_libri_dataset.py @@ -0,0 +1,132 @@ +from pathlib import Path +import unittest + +from torch import Tensor +import torchaudio +from voicefixer import Vocoder + +from training.datasets.hifi_libri_dataset import HifiLibriDataset, HifiLibriItem + + +class TestHifiLibriDataset(unittest.TestCase): + def setUp(self): + self.cache_dir = "datasets_cache" + self.dataset = HifiLibriDataset(cache_dir=self.cache_dir, cache=True) + self.vocoder_vf = Vocoder(44100) + + def test_init(self): + self.assertEqual(len(self.dataset.cutset), 129751) + + def test_get_cache_subdir_path(self): + idx = 1234 + expected_path = Path(self.cache_dir) / "cache-hifitts-librittsr" / "2000" + self.assertEqual(self.dataset.get_cache_subdir_path(idx), expected_path) + + def test_get_cache_file_path(self): + idx = 1234 + expected_path = ( + Path(self.cache_dir) / "cache-hifitts-librittsr" / "2000" / f"{idx}.pt" + ) + self.assertEqual(self.dataset.get_cache_file_path(idx), expected_path) + + def test_getitem(self): + # Take the hifi items from the beginning of the dataset + item = self.dataset[0] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "hifitts") + + # Convert mel spectrogram to waveform and save it to a file + # NOTE: Vocoder expects the mel spectrogram to be prepared in a specific way + # wav = self.vocoder_vf.forward(item.mel.permute((1, 0)).unsqueeze(0)) + # wav_path = Path(f"results/{item.id}.wav") + + # torchaudio.save(str(wav_path), wav, 44100) + + # Check that the cache file is created + cache_file = self.dataset.get_cache_file_path(0) + self.assertTrue(cache_file.exists()) + # Take the same id again to check if the cache is used + item = self.dataset[0] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "hifitts") + + item = self.dataset[10] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "hifitts") + # Check that the cache file is created + cache_file = self.dataset.get_cache_file_path(10) + self.assertTrue(cache_file.exists()) + + item = self.dataset[20] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "hifitts") + + # Take the libri items from the end of the dataset + item = self.dataset[len(self.dataset) - 20] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "libritts") + # Check that the cache file is created + cache_file = self.dataset.get_cache_file_path(len(self.dataset) - 20) + self.assertTrue(cache_file.exists()) + + item = self.dataset[len(self.dataset) - 10] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "libritts") + item = self.dataset[len(self.dataset) - 5] + self.assertIsInstance(item, HifiLibriItem) + self.assertEqual(item.dataset_type, "libritts") + + def test_collate_fn(self): + data = [self.dataset[0] for _ in range(10)] + collated = self.dataset.collate_fn(data) + self.assertIsInstance(collated, list) + self.assertIsInstance(collated[0], list) # ids + self.assertIsInstance(collated[1], list) # raw_texts + self.assertIsInstance(collated[2], Tensor) # speakers + self.assertIsInstance(collated[3], Tensor) # texts + self.assertIsInstance(collated[4], Tensor) # src_lens + self.assertIsInstance(collated[5], Tensor) # mels + self.assertIsInstance(collated[6], Tensor) # pitches + self.assertIsInstance(collated[7], list) # pitches_stat + self.assertIsInstance(collated[8], Tensor) # mel_lens + self.assertIsInstance(collated[9], Tensor) # langs + self.assertIsInstance(collated[10], Tensor) # attn_priors + self.assertIsInstance(collated[11], Tensor) # wavs + self.assertIsInstance(collated[12], Tensor) # energy + + def test_include_libri(self): + dataset_with_libri = HifiLibriDataset( + cache_dir="datasets_cache", + include_libri=True, + ) + dataset_without_libri = HifiLibriDataset( + cache_dir="datasets_cache", + include_libri=False, + ) + + # Check that the dataset with LibriTTS is larger than the dataset without LibriTTS + self.assertTrue(len(dataset_with_libri) > len(dataset_without_libri)) + + # Check that the dataset with LibriTTS includes items of type 'libritts' + libri_item = dataset_with_libri[len(dataset_with_libri) - 10] + self.assertIsInstance(libri_item, HifiLibriItem) + self.assertEqual(libri_item.dataset_type, "libritts") + + # Check that the dataset without LibriTTS does not include items of type 'libritts' + hifi_item = dataset_without_libri[len(dataset_without_libri) - 10] + self.assertIsInstance(hifi_item, HifiLibriItem) + self.assertEqual(hifi_item.dataset_type, "hifitts") + + def test_dur_filter(self): + # Test with a duration of 0.2 + self.assertFalse(self.dataset.dur_filter(0.2)) + # Test with a duration of 1.0 + self.assertTrue(self.dataset.dur_filter(1.0)) + # Test with a duration of 2.0 + self.assertTrue(self.dataset.dur_filter(2.0)) + # Test with a duration of 30.0 + self.assertFalse(self.dataset.dur_filter(30.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/datasets/tests/test_libritts_dataset_acoustic.py b/training/datasets/tests/test_libritts_dataset_acoustic.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee7a4ab9e8e3d68af484ba1a0004d2720c92d4b --- /dev/null +++ b/training/datasets/tests/test_libritts_dataset_acoustic.py @@ -0,0 +1,123 @@ +import os +import unittest + +import torch +from torch.utils.data import DataLoader + +from training.datasets import LibriTTSDatasetAcoustic + + +class TestLibriTTSDatasetAcoustic(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.lang = "en" + self.download = False + + self.dataset = LibriTTSDatasetAcoustic( + root="datasets_cache/LIBRITTS", + lang=self.lang, + download=self.download, + ) + + + # def test_len(self): + # self.assertEqual(len(self.dataset), 33236) + + + # def test_getitem(self): + # sample = self.dataset[0] + # self.assertEqual(sample["id"], "1034_121119_000001_000001") + # self.assertEqual(sample["speaker"], 1034) + + # self.assertEqual(sample["text"].shape, torch.Size([6])) + # self.assertEqual(sample["mel"].shape, torch.Size([100, 58])) + # self.assertEqual(sample["pitch"].shape, torch.Size([58])) + # self.assertEqual(sample["raw_text"], "The Law.") + # self.assertEqual(sample["normalized_text"], "The Law.") + # self.assertFalse(sample["pitch_is_normalized"]) + # self.assertEqual(sample["lang"], 3) + # self.assertEqual(sample["attn_prior"].shape, torch.Size([6, 58])) + # self.assertEqual(sample["wav"].shape, torch.Size([1, 14994])) + # self.assertEqual(sample["energy"].shape, torch.Size([1, 957])) + + # def test_cache_item(self): + # dataset = LibriTTSDatasetAcoustic( + # cache=True, + # ) + + # idxs = [0, 1, 1000, 1002, 2010] + + # for idx in idxs: + # # Get a sample from the dataset + # sample = dataset[idx] + + # cache_subdir_path = os.path.join(dataset.cache_dir, dataset.cache_subdir(idx)) + # cache_file = os.path.join(cache_subdir_path, f"{idx}.pt") + + # # Check if the data is in the cache + # self.assertTrue(os.path.exists(cache_file)) + + # # Load the data from the cache file + # cached_sample = torch.load(cache_file) + + # # Check if the cached data is the same as the original data + # for key in sample: + # if torch.is_tensor(sample[key]): + # self.assertTrue(torch.all(sample[key] == cached_sample[key])) + # else: + # self.assertEqual(sample[key], cached_sample[key]) + + + # def test_collate_fn(self): + # data = [ + # self.dataset[0], + # self.dataset[2], + # ] + + # # Call the collate_fn method + # result = self.dataset.collate_fn(data) + + # # Check the output + # self.assertEqual(len(result), 13) + + # # Check that all the batches are the same size + # for batch in result: + # self.assertEqual(len(batch), 2) + + + def test_normalize_pitch(self): + pitches = [ + torch.tensor([100.0, 200.0, 300.0]), + torch.tensor([150.0, 250.0, 350.0]), + ] + + result = self.dataset.normalize_pitch(pitches) + + expected_output = (100.0, 350.0, 225.0, 93.54143524169922) + + self.assertEqual(result, expected_output) + + # def test_dataloader(self): + # # Create a DataLoader from the dataset + # dataloader = DataLoader( + # self.dataset, + # batch_size=self.batch_size, + # shuffle=False, + # collate_fn=self.dataset.collate_fn, + # ) + + # iter_dataloader = iter(dataloader) + + # # Iterate over the DataLoader and check the output + # for _, items in enumerate([next(iter_dataloader), next(iter_dataloader)]): + # # items = batch[0] + + # # Check the batch size + # self.assertEqual(len(items), 13) + + # for it in items: + # self.assertEqual(len(it), self.batch_size) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/datasets/tests/test_libritts_dataset_vocoder.py b/training/datasets/tests/test_libritts_dataset_vocoder.py new file mode 100644 index 0000000000000000000000000000000000000000..79297dd163f6c688ea1ba816a30e89b802cf52d4 --- /dev/null +++ b/training/datasets/tests/test_libritts_dataset_vocoder.py @@ -0,0 +1,67 @@ +import unittest + +import torch +from torch.utils.data import DataLoader + +from training.datasets import LibriTTSDatasetVocoder + + +class TestLibriTTSDatasetAcoustic(unittest.TestCase): + def setUp(self): + self.batch_size = 2 + self.lang = "en" + self.download = False + + self.dataset = LibriTTSDatasetVocoder( + root="datasets_cache/LIBRITTS", + batch_size=self.batch_size, + download=self.download, + ) + + def test_len(self): + self.assertEqual(len(self.dataset), 33236) + + def test_getitem(self): + sample = self.dataset[0] + self.assertEqual(sample["mel"].shape, torch.Size([100, 64])) + self.assertEqual(sample["audio"].shape, torch.Size([16384])) + self.assertEqual(sample["speaker_id"], 1034) + + def test_collate_fn(self): + data = [ + self.dataset[0], + self.dataset[2], + ] + + # Call the collate_fn method + result = self.dataset.collate_fn(data) + + # Check the output + self.assertEqual(len(result), 4) + + # Check that all the batches are the same size + for batch in result: + self.assertEqual(len(batch), self.batch_size) + + def test_dataloader(self): + # Create a DataLoader from the dataset + dataloader = DataLoader( + self.dataset, + batch_size=self.batch_size, + shuffle=False, + collate_fn=self.dataset.collate_fn, + ) + + iter_dataloader = iter(dataloader) + + # Iterate over the DataLoader and check the output + for _, items in enumerate([next(iter_dataloader), next(iter_dataloader)]): + # Check the batch size + self.assertEqual(len(items), 4) + + for it in items: + self.assertEqual(len(it), self.batch_size) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/datasets/tests/test_libritts_r.py b/training/datasets/tests/test_libritts_r.py new file mode 100644 index 0000000000000000000000000000000000000000..026d12814e01189d4687bda272e3da1b9ff91dff --- /dev/null +++ b/training/datasets/tests/test_libritts_r.py @@ -0,0 +1,111 @@ +import os +import unittest + +from training.datasets.libritts_r import LIBRITTS_R, load_libritts_item + + +class TestLibriTTS(unittest.TestCase): + def setUp(self): + # Set up any necessary values for the tests + self.fileid = "1061_146197_000015_000000" + self.path = "datasets_cache/LIBRITTS/LibriTTS/train-clean-360" + self.ext_audio = ".wav" + self.ext_original_txt = ".original.txt" + self.ext_normalized_txt = ".normalized.txt" + + def test_load_libritts_item(self): + # Test the load_libritts_item function + waveform, sample_rate, original_text, normalized_text, speaker_id, chapter_id, utterance_id = load_libritts_item( + self.fileid, + self.path, + self.ext_audio, + self.ext_original_txt, + self.ext_normalized_txt, + ) + + base_path = os.path.join( + self.path, + f"{speaker_id}", + f"{chapter_id}", + ) + + # Check that the files were created + self.assertTrue( + os.path.exists( + os.path.join( + base_path, + self.fileid + self.ext_original_txt, + ), + ), + ) + self.assertTrue( + os.path.exists( + os.path.join( + base_path, + self.fileid + self.ext_normalized_txt, + ), + ), + ) + + def test_selected_speaker_ids(self): + # Initialize the dataset with selected speaker IDs + dataset = LIBRITTS_R(root="datasets_cache/LIBRITTS", url="train-clean-100", selected_speaker_ids=[19, 26]) + + # Iterate over the dataset and check the speaker IDs + for _, _, _, _, speaker_id, _, _ in dataset: + # Assert that the speaker ID is in the list of selected speaker IDs + self.assertIn(speaker_id, [19, 26]) + + def test_max_audio_length(self): + # Initialize the dataset with a maximum audio length + dataset = LIBRITTS_R( + root="datasets_cache/LIBRITTS", + url="train-clean-100", + max_audio_length=3.0, + selected_speaker_ids=[19, 26], + ) + + # Iterate over the dataset and check the audio lengths + for waveform, sample_rate, _, _, speaker_id, _, _ in dataset: + # Get the duration of the waveform in seconds + duration = waveform.shape[1] / sample_rate + + # Assert that the speaker ID is in the list of selected speaker IDs + self.assertIn(speaker_id, [19, 26]) + # Assert that the duration is less than or equal to the maximum length + self.assertLessEqual(duration, 3.0) + + def test_min_audio_length(self): + # Initialize the dataset with a minimum audio length + dataset = LIBRITTS_R( + root="datasets_cache/LIBRITTS", + url="train-clean-100", + min_audio_length=30.0, + ) + + # Iterate over the dataset and check the audio lengths + for waveform, sample_rate, _, _, _, _, _ in dataset: + # Get the duration of the waveform in seconds + duration = waveform.shape[1] / sample_rate + + # Assert that the duration is greater than or equal to the minimum length + self.assertGreaterEqual(duration, 30.0) + + # Add any other assertions you want to make about the return values + def tearDown(self): + speaker_id, chapter_id, _, _ = self.fileid.split("_") + + normalized_text_filename = self.fileid + self.ext_normalized_txt + normalized_text_path = os.path.join(self.path, speaker_id, chapter_id, normalized_text_filename) + + original_text_filename = self.fileid + self.ext_original_txt + original_text_path = os.path.join(self.path, speaker_id, chapter_id, original_text_filename) + + # Clean up any created files after tests are done + if os.path.exists(normalized_text_path): + os.remove(normalized_text_path) + if os.path.exists(original_text_path): + os.remove(original_text_path) + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/__init__.py b/training/loss/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..abd4ef97d7a8589a860fa0a26e5a717f6aade319 --- /dev/null +++ b/training/loss/__init__.py @@ -0,0 +1,12 @@ +from .bin_loss import BinLoss +from .delightful_tts_loss import DelightfulTTSLoss +from .fast_speech_2_loss_gen import FastSpeech2LossGen +from .forward_sum_loss import ForwardSumLoss +from .hifi_loss import DiscriminatorLoss, FeatureMatchingLoss, GeneratorLoss +from .log_stft_magnitude_loss import LogSTFTMagnitudeLoss + +# from .metrics import Metrics +from .multi_resolution_stft_loss import MultiResolutionSTFTLoss +from .spectral_convergence_loss import SpectralConvergengeLoss +from .stft_loss import STFTLoss +from .univnet_loss import UnivnetLoss diff --git a/training/loss/bin_loss.py b/training/loss/bin_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..e93e83161d9a1269915ce0b1cd4137c81fe5ba57 --- /dev/null +++ b/training/loss/bin_loss.py @@ -0,0 +1,35 @@ +import torch +from torch.nn import Module + + +class BinLoss(Module): + r"""Binary cross-entropy loss for hard and soft attention. + + Attributes + None + + Methods + forward: Computes the binary cross-entropy loss for hard and soft attention. + + """ + + def __init__(self): + super().__init__() + + def forward( + self, hard_attention: torch.Tensor, soft_attention: torch.Tensor, + ) -> torch.Tensor: + r"""Computes the binary cross-entropy loss for hard and soft attention. + + Args: + hard_attention (torch.Tensor): A binary tensor indicating the hard attention. + soft_attention (torch.Tensor): A tensor containing the soft attention probabilities. + + Returns: + torch.Tensor: The binary cross-entropy loss. + + """ + log_sum = torch.log( + torch.clamp(soft_attention[hard_attention == 1], min=1e-12), + ).sum() + return -log_sum / hard_attention.sum() diff --git a/training/loss/delightful_tts_loss.py b/training/loss/delightful_tts_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..f106d01e610670c6a131afbe233a4c30fa8647dc --- /dev/null +++ b/training/loss/delightful_tts_loss.py @@ -0,0 +1,308 @@ +from typing import Optional + +from piq import SSIMLoss +import torch +from torch import Tensor, nn +from torch.nn import functional + +from models.config import AcousticModelConfigType +from training.loss.utils import sample_wise_min_max + + +# from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1 +def sequence_mask(sequence_length: Tensor, max_len: Optional[int] = None) -> Tensor: + """Create a sequence mask for filtering padding in a sequence tensor. + + Args: + sequence_length (torch.tensor): Sequence lengths. + max_len (int, Optional): Maximum sequence length. Defaults to None. + + Shapes: + - mask: :math:`[B, T_max]` + """ + max_len_ = max_len if max_len is not None else sequence_length.max().item() + + seq_range = torch.arange(max_len_, dtype=sequence_length.dtype, device=sequence_length.device) + + # B x T_max + return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1) + + +class ForwardSumLoss(nn.Module): + r"""A class used to compute the forward sum loss. + + Attributes: + log_softmax (torch.nn.LogSoftmax): The log softmax function applied along dimension 3. + ctc_loss (torch.nn.CTCLoss): The CTC loss function with zero infinity set to True. + blank_logprob (int): The log probability of a blank, default is -1. + + Methods: + forward(attn_logprob: Tensor, in_lens: Tensor, out_lens: Tensor) + Compute the forward sum loss. + """ + + def __init__(self, blank_logprob: int = -1): + r"""Constructs all the necessary attributes for the ForwardSumLoss object. + + Args: + blank_logprob (int, optional): The log probability of a blank (default is -1). + """ + super().__init__() + self.log_softmax = torch.nn.LogSoftmax(dim=3) + self.ctc_loss = torch.nn.CTCLoss(zero_infinity=True) + self.blank_logprob = blank_logprob + + def forward(self, attn_logprob: Tensor, in_lens: Tensor, out_lens: Tensor): + r"""Compute the forward sum loss. + + Args: + attn_logprob (Tensor): The attention log probabilities. + in_lens (Tensor): The input lengths. + out_lens (Tensor): The output lengths. + + Returns: + total_loss (float): The total loss computed. + """ + key_lens = in_lens + query_lens = out_lens + attn_logprob_padded = functional.pad(input=attn_logprob, pad=(1, 0), value=self.blank_logprob) + + total_loss = 0.0 + for bid in range(attn_logprob.shape[0]): + target_seq = torch.arange(1, key_lens[bid].item() + 1).unsqueeze(0) + curr_logprob = attn_logprob_padded[bid].permute(1, 0, 2)[: query_lens[bid], :, : key_lens[bid] + 1] + + curr_logprob = self.log_softmax(curr_logprob[None])[0] + loss = self.ctc_loss( + curr_logprob, + target_seq, + input_lengths=query_lens[bid : bid + 1], + target_lengths=key_lens[bid : bid + 1], + ) + total_loss = total_loss + loss + + total_loss = total_loss / attn_logprob.shape[0] + return total_loss + + +class DelightfulTTSLoss(nn.Module): + r"""A class used to compute the delightful TTS loss. + + Attributes: + mse_loss (nn.MSELoss): The mean squared error loss function. + mae_loss (nn.L1Loss): The mean absolute error loss function. + forward_sum_loss (ForwardSumLoss): The forward sum loss function. + mel_loss_alpha (float): The weight for the mel loss. + aligner_loss_alpha (float): The weight for the aligner loss. + pitch_loss_alpha (float): The weight for the pitch loss. + energy_loss_alpha (float): The weight for the energy loss. + u_prosody_loss_alpha (float): The weight for the u prosody loss. + p_prosody_loss_alpha (float): The weight for the p prosody loss. + dur_loss_alpha (float): The weight for the duration loss. + binary_alignment_loss_alpha (float): The weight for the binary alignment loss. + + Methods: + _binary_alignment_loss(alignment_hard: Tensor, alignment_soft: Tensor) + Compute the binary alignment loss. + forward( + mel_output: Tensor, + mel_target: Tensor, + mel_lens: Tensor, + dur_output: Tensor, + dur_target: Tensor, + pitch_output: Tensor, + pitch_target: Tensor, + energy_output: Tensor, + energy_target: Tensor, + src_lens: Tensor, + p_prosody_ref: Tensor, + p_prosody_pred: Tensor, + u_prosody_ref: Tensor, + u_prosody_pred: Tensor, + aligner_logprob: Tensor, + aligner_hard: Tensor, + aligner_soft: Tensor, + binary_loss_weight: Optional[Tensor] = None, + ) + Compute the delightful TTS loss. + """ + + def __init__(self, config: AcousticModelConfigType): + r"""Constructs all the necessary attributes for the DelightfulTTSLoss object. + + Args: + config (AcousticModelConfigType): Configuration parameters for the loss function. + """ + super().__init__() + + self.mse_loss = nn.MSELoss() + self.mae_loss = nn.L1Loss() + self.forward_sum_loss = ForwardSumLoss() + self.ssim_loss = SSIMLoss() + + self.mel_loss_alpha = config.loss.mel_loss_alpha + self.ssim_loss_alpha = config.loss.ssim_loss_alpha + self.aligner_loss_alpha = config.loss.aligner_loss_alpha + self.pitch_loss_alpha = config.loss.pitch_loss_alpha + self.energy_loss_alpha = config.loss.energy_loss_alpha + self.u_prosody_loss_alpha = config.loss.u_prosody_loss_alpha + self.p_prosody_loss_alpha = config.loss.p_prosody_loss_alpha + self.dur_loss_alpha = config.loss.dur_loss_alpha + self.binary_alignment_loss_alpha = config.loss.binary_align_loss_alpha + + @staticmethod + def _binary_alignment_loss(alignment_hard: Tensor, alignment_soft: Tensor) -> Tensor: + """Binary loss that forces soft alignments to match the hard alignments as + explained in `https://arxiv.org/pdf/2108.10447.pdf`. + + Args: + alignment_hard (Tensor): The hard alignment tensor. + alignment_soft (Tensor): The soft alignment tensor. + + Returns: + loss (float): The computed binary alignment loss. + """ + log_sum = torch.log(torch.clamp(alignment_soft[alignment_hard == 1], min=1e-12)).sum() + return -log_sum / alignment_hard.sum() + + def forward( + self, + mel_output: Tensor, + mel_target: Tensor, + mel_lens: Tensor, + dur_output: Tensor, + dur_target: Tensor, + pitch_output: Tensor, + pitch_target: Tensor, + energy_output: Tensor, + energy_target: Tensor, + src_lens: Tensor, + p_prosody_ref: Tensor, + p_prosody_pred: Tensor, + u_prosody_ref: Tensor, + u_prosody_pred: Tensor, + aligner_logprob: Tensor, + aligner_hard: Tensor, + aligner_soft: Tensor, + ): + r"""Compute the delightful TTS loss. + + Args: + mel_output (Tensor): The mel output tensor. + mel_target (Tensor): The mel target tensor. + mel_lens (Tensor): The mel lengths tensor. + dur_output (Tensor): The duration output tensor. + dur_target (Tensor): The duration target tensor. + pitch_output (Tensor): The pitch output tensor. + pitch_target (Tensor): The pitch target tensor. + energy_output (Tensor): The energy output tensor. + energy_target (Tensor): The energy target tensor. + src_lens (Tensor): The source lengths tensor. + p_prosody_ref (Tensor): The p prosody reference tensor. + p_prosody_pred (Tensor): The p prosody prediction tensor. + u_prosody_ref (Tensor): The u prosody reference tensor. + u_prosody_pred (Tensor): The u prosody prediction tensor. + aligner_logprob (Tensor): The aligner log probabilities tensor. + aligner_hard (Tensor): The hard aligner tensor. + aligner_soft (Tensor): The soft aligner tensor. + + Returns: + loss_dict (Tupple): A dictionary containing all the loss values. + + Shapes: + - mel_output: :math:`(B, C_mel, T_mel)` + - mel_target: :math:`(B, C_mel, T_mel)` + - mel_lens: :math:`(B)` + - dur_output: :math:`(B, T_src)` + - dur_target: :math:`(B, T_src)` + - pitch_output: :math:`(B, 1, T_src)` + - pitch_target: :math:`(B, 1, T_src)` + - energy_output: :math:`(B, 1, T_src)` + - energy_target: :math:`(B, 1, T_src)` + - src_lens: :math:`(B)` + - p_prosody_ref: :math:`(B, T_src, 4)` + - p_prosody_pred: :math:`(B, T_src, 4)` + - u_prosody_ref: :math:`(B, 1, 256) + - u_prosody_pred: :math:`(B, 1, 256) + - aligner_logprob: :math:`(B, 1, T_mel, T_src)` + - aligner_hard: :math:`(B, T_mel, T_src)` + - aligner_soft: :math:`(B, T_mel, T_src)` + """ + src_mask = sequence_mask(src_lens).to(mel_output.device) # (B, T_src) + mel_mask = sequence_mask(mel_lens).to(mel_output.device) # (B, T_mel) + + dur_target.requires_grad = False + mel_target.requires_grad = False + pitch_target.requires_grad = False + + mel_predictions_normalized = sample_wise_min_max(mel_output).float().to(mel_output.device) + mel_targets_normalized = sample_wise_min_max(mel_target).float().to(mel_target.device) + + masked_mel_predictions = mel_output.masked_select(mel_mask[:, None]) + mel_targets = mel_target.masked_select(mel_mask[:, None]) + mel_loss = self.mae_loss(masked_mel_predictions, mel_targets) * self.mel_loss_alpha + + ssim_loss: torch.Tensor = self.ssim_loss( + mel_predictions_normalized.unsqueeze(1), mel_targets_normalized.unsqueeze(1), + ) * self.ssim_loss_alpha + + if ssim_loss.item() > 1.0 or ssim_loss.item() < 0.0: + print( + f"Overflow in ssim loss detected, which was {ssim_loss.item()}, setting to 1.0", + ) + ssim_loss = torch.tensor([1.0], device=mel_output.device) + + p_prosody_ref = p_prosody_ref.detach() + p_prosody_loss = self.mae_loss( + p_prosody_ref.masked_select(src_mask.unsqueeze(-1)), + p_prosody_pred.masked_select(src_mask.unsqueeze(-1)), + ) * self.p_prosody_loss_alpha + + u_prosody_ref = u_prosody_ref.detach() + u_prosody_loss = self.mae_loss(u_prosody_ref, u_prosody_pred) * self.u_prosody_loss_alpha + + duration_loss = self.mse_loss(dur_output, dur_target) * self.dur_loss_alpha + + pitch_output = pitch_output.masked_select(src_mask[:, None]) + pitch_target = pitch_target.masked_select(src_mask[:, None]) + pitch_loss = self.mse_loss(pitch_output, pitch_target) * self.pitch_loss_alpha + + energy_output = energy_output.masked_select(src_mask[:, None]) + energy_target = energy_target.masked_select(src_mask[:, None]) + energy_loss = self.mse_loss(energy_output, energy_target) * self.energy_loss_alpha + + forward_sum_loss = self.forward_sum_loss( + aligner_logprob, + src_lens, + mel_lens, + ) * self.aligner_loss_alpha + + binary_alignment_loss = self._binary_alignment_loss( + aligner_hard, + aligner_soft, + ) * self.binary_alignment_loss_alpha + + total_loss = ( + mel_loss + + ssim_loss + + duration_loss + + u_prosody_loss + + p_prosody_loss + + pitch_loss + + forward_sum_loss + + binary_alignment_loss + + energy_loss + ) + + return ( + total_loss, + mel_loss, + ssim_loss, + duration_loss, + u_prosody_loss, + p_prosody_loss, + pitch_loss, + forward_sum_loss, + binary_alignment_loss, + energy_loss, + ) diff --git a/training/loss/fast_speech_2_loss_gen.py b/training/loss/fast_speech_2_loss_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..81a14522e05aee3680ef314f9a8e9ecf050c8e73 --- /dev/null +++ b/training/loss/fast_speech_2_loss_gen.py @@ -0,0 +1,234 @@ +from typing import Tuple + +from piq import SSIMLoss +import torch +from torch import nn +from torch.nn import Module + +from training.loss.bin_loss import BinLoss +from training.loss.forward_sum_loss import ForwardSumLoss +from training.loss.utils import sample_wise_min_max + + +class FastSpeech2LossGen(Module): + def __init__( + self, + bin_warmup: bool = True, + binarization_loss_enable_steps: int = 1260, + binarization_loss_warmup_steps: int = 700, + ): + r"""Initializes the FastSpeech2LossGen module. + + Args: + bin_warmup (bool, optional): Whether to use binarization warmup. Defaults to True. NOTE: Switch this off if you preload the model with a checkpoint that has already passed the warmup phase. + binarization_loss_enable_steps (int, optional): Number of steps to enable the binarization loss. Defaults to 1260. + binarization_loss_warmup_steps (int, optional): Number of warmup steps for the binarization loss. Defaults to 700. + """ + super().__init__() + + self.mse_loss = nn.MSELoss() + self.mae_loss = nn.L1Loss() + self.ssim_loss = SSIMLoss() + self.sum_loss = ForwardSumLoss() + self.bin_loss = BinLoss() + + self.bin_warmup = bin_warmup + self.binarization_loss_enable_steps = binarization_loss_enable_steps + self.binarization_loss_warmup_steps = binarization_loss_warmup_steps + + def forward( + self, + src_masks: torch.Tensor, + mel_masks: torch.Tensor, + mel_targets: torch.Tensor, + mel_predictions: torch.Tensor, + log_duration_predictions: torch.Tensor, + u_prosody_ref: torch.Tensor, + u_prosody_pred: torch.Tensor, + p_prosody_ref: torch.Tensor, + p_prosody_pred: torch.Tensor, + durations: torch.Tensor, + pitch_predictions: torch.Tensor, + p_targets: torch.Tensor, + attn_logprob: torch.Tensor, + attn_soft: torch.Tensor, + attn_hard: torch.Tensor, + step: int, + src_lens: torch.Tensor, + mel_lens: torch.Tensor, + energy_pred: torch.Tensor, + energy_target: torch.Tensor, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + ]: + r"""Computes the loss for the FastSpeech2 model. + + Args: + src_masks (torch.Tensor): Mask for the source sequence. + mel_masks (torch.Tensor): Mask for the mel-spectrogram. + mel_targets (torch.Tensor): Target mel-spectrogram. + mel_predictions (torch.Tensor): Predicted mel-spectrogram. + log_duration_predictions (torch.Tensor): Predicted log-duration. + u_prosody_ref (torch.Tensor): Reference unvoiced prosody. + u_prosody_pred (torch.Tensor): Predicted unvoiced prosody. + p_prosody_ref (torch.Tensor): Reference voiced prosody. + p_prosody_pred (torch.Tensor): Predicted voiced prosody. + durations (torch.Tensor): Ground-truth durations. + pitch_predictions (torch.Tensor): Predicted pitch. + p_targets (torch.Tensor): Ground-truth pitch. + attn_logprob (torch.Tensor): Log-probability of attention. + attn_soft (torch.Tensor): Soft attention. + attn_hard (torch.Tensor): Hard attention. + step (int): Current training step. + src_lens (torch.Tensor): Lengths of the source sequences. + mel_lens (torch.Tensor): Lengths of the mel-spectrograms. + energy_pred (torch.Tensor): Predicted energy. + energy_target (torch.Tensor): Ground-truth energy. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: The total loss and its components. + + Note: + Here is the description of the returned loss components: + `total_loss`: This is the total loss computed as the sum of all the other losses. + `mel_loss`: This is the mean absolute error (MAE) loss between the predicted and target mel-spectrograms. It measures how well the model predicts the mel-spectrograms. + `sc_mag_loss`: This is the spectral convergence loss between the predicted and target mel-spectrograms. It measures how well the model predicts the mel-spectrograms in terms of their spectral structure. + `log_mag_loss`: This is the log STFT magnitude loss between the predicted and target mel-spectrograms. It measures how well the model predicts the mel-spectrograms in terms of their spectral structure. + `ssim_loss`: This is the Structural Similarity Index (SSIM) loss between the predicted and target mel-spectrograms. It measures the similarity between the two mel-spectrograms in terms of their structure, contrast, and luminance. + `duration_loss`: This is the mean squared error (MSE) loss between the predicted and target log-durations. It measures how well the model predicts the durations of the phonemes. + `u_prosody_loss`: This is the MAE loss between the predicted and reference unvoiced prosody. It measures how well the model predicts the prosody (rhythm, stress, and intonation) of the unvoiced parts of the speech. + `p_prosody_loss`: This is the MAE loss between the predicted and reference voiced prosody. It measures how well the model predicts the prosody of the voiced parts of the speech. + `pitch_loss`: This is the MSE loss between the predicted and target pitch. It measures how well the model predicts the pitch of the speech. + `ctc_loss`: This is the Connectionist Temporal Classification (CTC) loss computed from the log-probability of attention and the lengths of the source sequences and mel-spectrograms. It measures how well the model aligns the input and output sequences. + `bin_loss`: This is the binarization loss computed from the hard and soft attention. It measures how well the model learns to attend to the correct parts of the input sequence. + `energy_loss`: This is the MSE loss between the predicted and target energy. It measures how well the model predicts the energy of the speech. + """ + log_duration_targets = torch.log(durations.float() + 1).to(src_masks.device) + + log_duration_targets.requires_grad = False + mel_targets.requires_grad = False + p_targets.requires_grad = False + energy_target.requires_grad = False + + log_duration_predictions = log_duration_predictions.masked_select(~src_masks) + log_duration_targets = log_duration_targets.masked_select(~src_masks) + + mel_masks_expanded = mel_masks.unsqueeze(1) + + mel_predictions_normalized = ( + sample_wise_min_max(mel_predictions).float().to(mel_predictions.device) + ) + mel_targets_normalized = ( + sample_wise_min_max(mel_targets).float().to(mel_predictions.device) + ) + + ssim_loss: torch.Tensor = self.ssim_loss( + mel_predictions_normalized.unsqueeze(1), + mel_targets_normalized.unsqueeze(1), + ) + + if ssim_loss.item() > 1.0 or ssim_loss.item() < 0.0: + ssim_loss = torch.tensor([1.0], device=mel_predictions.device) + + masked_mel_predictions = mel_predictions.masked_select(~mel_masks_expanded) + + masked_mel_targets = mel_targets.masked_select(~mel_masks_expanded) + + mel_loss: torch.Tensor = self.mae_loss( + masked_mel_predictions, + masked_mel_targets, + ) + + p_prosody_ref = p_prosody_ref.permute((0, 2, 1)) + p_prosody_pred = p_prosody_pred.permute((0, 2, 1)) + + p_prosody_ref = p_prosody_ref.masked_fill(src_masks.unsqueeze(1), 0.0) + p_prosody_pred = p_prosody_pred.masked_fill(src_masks.unsqueeze(1), 0.0) + + p_prosody_ref = p_prosody_ref.detach() + + p_prosody_loss: torch.Tensor = 0.5 * self.mae_loss( + p_prosody_ref.masked_select(~src_masks.unsqueeze(1)), + p_prosody_pred.masked_select(~src_masks.unsqueeze(1)), + ) + + u_prosody_ref = u_prosody_ref.detach() + u_prosody_loss: torch.Tensor = 0.5 * self.mae_loss( + u_prosody_ref, + u_prosody_pred, + ) + + duration_loss: torch.Tensor = self.mse_loss( + log_duration_predictions, + log_duration_targets, + ) + + pitch_predictions = pitch_predictions.masked_select(~src_masks) + p_targets = p_targets.masked_select(~src_masks) + + pitch_loss: torch.Tensor = self.mse_loss(pitch_predictions, p_targets) + + ctc_loss: torch.Tensor = self.sum_loss( + attn_logprob=attn_logprob, + in_lens=src_lens, + out_lens=mel_lens, + ) + + if self.bin_warmup: + if step < self.binarization_loss_enable_steps: + bin_loss_weight = 0.0 + else: + bin_loss_weight = ( + min( + (step - self.binarization_loss_enable_steps) + / self.binarization_loss_warmup_steps, + 1.0, + ) + * 1.0 + ) + + bin_loss: torch.Tensor = ( + self.bin_loss(hard_attention=attn_hard, soft_attention=attn_soft) + * bin_loss_weight + ) + else: + bin_loss: torch.Tensor = self.bin_loss( + hard_attention=attn_hard, + soft_attention=attn_soft, + ) + + energy_loss: torch.Tensor = self.mse_loss(energy_pred, energy_target) + + total_loss = ( + mel_loss + + duration_loss + + u_prosody_loss + + p_prosody_loss + + ssim_loss + + pitch_loss + + ctc_loss + + bin_loss + + energy_loss + ) + + return ( + total_loss, + mel_loss, + ssim_loss, + duration_loss, + u_prosody_loss, + p_prosody_loss, + pitch_loss, + ctc_loss, + bin_loss, + energy_loss, + ) diff --git a/training/loss/forward_sum_loss.py b/training/loss/forward_sum_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..dc9d7612ed6542b64ea2821c870389451ed93998 --- /dev/null +++ b/training/loss/forward_sum_loss.py @@ -0,0 +1,66 @@ +import torch +from torch import nn +from torch.nn import Module +from torch.nn import functional as F + + +class ForwardSumLoss(Module): + r"""Computes the forward sum loss for sequence-to-sequence models with attention. + + Args: + blank_logprob (float): The log probability of the blank symbol. Default: -1. + + Attributes: + log_softmax (nn.LogSoftmax): The log softmax function. + ctc_loss (nn.CTCLoss): The CTC loss function. + blank_logprob (float): The log probability of the blank symbol. + + Methods: + forward: Computes the forward sum loss for sequence-to-sequence models with attention. + + """ + + def __init__(self, blank_logprob: float = -1): + super().__init__() + self.log_softmax = nn.LogSoftmax(dim=3) + self.ctc_loss = nn.CTCLoss(zero_infinity=True) + self.blank_logprob = blank_logprob + + def forward( + self, attn_logprob: torch.Tensor, in_lens: torch.Tensor, out_lens: torch.Tensor, + ) -> float: + r"""Computes the forward sum loss for sequence-to-sequence models with attention. + + Args: + attn_logprob (torch.Tensor): The attention log probabilities of shape (batch_size, max_out_len, max_in_len). + in_lens (torch.Tensor): The input lengths of shape (batch_size,). + out_lens (torch.Tensor): The output lengths of shape (batch_size,). + + Returns: + float: The forward sum loss. + + """ + key_lens = in_lens + query_lens = out_lens + attn_logprob_padded = F.pad( + input=attn_logprob, pad=(1, 0), value=self.blank_logprob, + ) + + total_loss = 0.0 + for bid in range(attn_logprob.shape[0]): + target_seq = torch.arange(1, int(key_lens[bid]) + 1).unsqueeze(0) + curr_logprob = attn_logprob_padded[bid].permute(1, 0, 2)[ + : int(query_lens[bid]), :, : int(key_lens[bid]) + 1, + ] + + curr_logprob = self.log_softmax(curr_logprob[None])[0] + loss = self.ctc_loss( + curr_logprob, + target_seq, + input_lengths=query_lens[bid : bid + 1], + target_lengths=key_lens[bid : bid + 1], + ) + total_loss += loss + + total_loss /= attn_logprob.shape[0] + return total_loss diff --git a/training/loss/hifi_loss.py b/training/loss/hifi_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..44a91cb7a1d0abc1df5db74624ad1eafef5b8178 --- /dev/null +++ b/training/loss/hifi_loss.py @@ -0,0 +1,52 @@ +from typing import List + +import torch +from torch import Tensor +from torch.nn.modules.loss import _Loss + + +class DiscriminatorLoss(_Loss): + """Discriminator Loss module""" + + def forward( + self, + disc_real_outputs: List[Tensor], + disc_generated_outputs: List[Tensor], + ): + loss = 0 + r_losses = [] + g_losses = [] + for dr, dg in zip(disc_real_outputs, disc_generated_outputs): + r_loss = torch.mean((1 - dr) ** 2) + g_loss = torch.mean(dg**2) + loss += r_loss + g_loss + r_losses.append(r_loss.item()) + g_losses.append(g_loss.item()) + + return loss, r_losses, g_losses + + +class FeatureMatchingLoss(_Loss): + """Feature Matching Loss module""" + + def forward(self, fmap_r: List[Tensor], fmap_g: List[Tensor]): + loss = 0 + for dr, dg in zip(fmap_r, fmap_g): + for rl, gl in zip(dr, dg): + loss += torch.mean(torch.abs(rl - gl)) + + return loss * 2 + + +class GeneratorLoss(_Loss): + """Generator Loss module""" + + def forward(self, disc_outputs: List[Tensor]): + loss = 0 + gen_losses = [] + for dg in disc_outputs: + l = torch.mean((1 - dg) ** 2) + gen_losses.append(l) + loss += l + + return loss, gen_losses diff --git a/training/loss/log_stft_magnitude_loss.py b/training/loss/log_stft_magnitude_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..2369996c7fe7f67ccbcefa46fccec4a30d8f8227 --- /dev/null +++ b/training/loss/log_stft_magnitude_loss.py @@ -0,0 +1,32 @@ +import torch +from torch.nn import Module +import torch.nn.functional as F + + +class LogSTFTMagnitudeLoss(Module): + r"""Log STFT magnitude loss module. + Log STFT magnitude loss is a loss function that is commonly used in speech and audio signal processing tasks, such as speech enhancement and source separation. It is a modification of the spectral convergence loss, which measures the similarity between two magnitude spectrograms. + + The log STFT magnitude loss is calculated as the mean squared error between the logarithm of the predicted and groundtruth magnitude spectrograms. The logarithm is applied to the magnitude spectrograms to convert them to a decibel scale, which is more perceptually meaningful than the linear scale. The mean squared error is used to penalize large errors between the predicted and groundtruth spectrograms. + """ + + def __init__(self): + r"""Initilize los STFT magnitude loss module.""" + super().__init__() + + def forward(self, x_mag: torch.Tensor, y_mag: torch.Tensor) -> torch.Tensor: + r"""Calculate forward propagation. + + Args: + x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). + y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). + + Returns: + Tensor: Log STFT magnitude loss value. + """ + # Ensure that x_mag and y_mag have the same size along dimension 1 + min_len = min(x_mag.shape[1], y_mag.shape[1]) + x_mag = x_mag[:, :min_len] + y_mag = y_mag[:, :min_len] + + return F.l1_loss(torch.log(y_mag), torch.log(x_mag)) diff --git a/training/loss/metrics.py b/training/loss/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..8d037090d498f43e16a2cfcf2786b0a8aea7cab8 --- /dev/null +++ b/training/loss/metrics.py @@ -0,0 +1,393 @@ +from dataclasses import dataclass +from typing import Optional + +import librosa +import matplotlib.pyplot as plt +import numpy as np +import torch +from torch import nn +import torchaudio.transforms as T +from torchmetrics.audio import ( + ComplexScaleInvariantSignalNoiseRatio, + ScaleInvariantSignalDistortionRatio, + ScaleInvariantSignalNoiseRatio, + SpeechReverberationModulationEnergyRatio, +) + +from models.config import PreprocessingConfig, PreprocessingConfigUnivNet, get_lang_map +from training.preprocess.audio_processor import AudioProcessor + + +@dataclass +class MetricsResult: + r"""A data class that holds the results of the computed metrics. + + Attributes: + energy (torch.Tensor): The energy loss ratio. + si_sdr (torch.Tensor): The scale-invariant signal-to-distortion ratio. + si_snr (torch.Tensor): The scale-invariant signal-to-noise ratio. + c_si_snr (torch.Tensor): The complex scale-invariant signal-to-noise ratio. + mcd (torch.Tensor): The Mel cepstral distortion. + spec_dist (torch.Tensor): The spectrogram distance. + f0_rmse (float): The F0 RMSE. + jitter (float): The jitter. + shimmer (float): The shimmer. + """ + + energy: torch.Tensor + si_sdr: torch.Tensor + si_snr: torch.Tensor + c_si_snr: torch.Tensor + mcd: torch.Tensor + spec_dist: torch.Tensor + f0_rmse: float + jitter: float + shimmer: float + + +class Metrics: + r"""A class that computes various audio metrics. + + Args: + lang (str): language parameter. Defaults to "en". + preprocess_config (Optional[PreprocessingConfig]): The preprocessing configuration. Defaults to None. + + Attributes: + hop_length (int): The hop length for the STFT. + filter_length (int): The filter length for the STFT. + mel_fmin (int): The minimum frequency for the Mel scale. + win_length (int): The window length for the STFT. + audio_processor (AudioProcessor): The audio processor. + mse_loss (nn.MSELoss): The mean squared error loss. + si_sdr (ScaleInvariantSignalDistortionRatio): The scale-invariant signal-to-distortion ratio. + si_snr (ScaleInvariantSignalNoiseRatio): The scale-invariant signal-to-noise ratio. + c_si_snr (ComplexScaleInvariantSignalNoiseRatio): The complex scale-invariant signal-to-noise ratio. + """ + + def __init__( + self, + lang: str = "en", + preprocess_config: Optional[PreprocessingConfig] = None, + ): + lang_map = get_lang_map(lang) + preprocess_config = preprocess_config or PreprocessingConfigUnivNet( + lang_map.processing_lang_type, + ) + + self.hop_length = preprocess_config.stft.hop_length + self.filter_length = preprocess_config.stft.filter_length + self.mel_fmin = preprocess_config.stft.mel_fmin + self.win_length = preprocess_config.stft.win_length + self.sample_rate = preprocess_config.sampling_rate + + self.audio_processor = AudioProcessor() + self.mse_loss = nn.MSELoss() + self.si_sdr = ScaleInvariantSignalDistortionRatio() + self.si_snr = ScaleInvariantSignalNoiseRatio() + self.c_si_snr = ComplexScaleInvariantSignalNoiseRatio(zero_mean=False) + self.reverb_modulation_energy_ratio = SpeechReverberationModulationEnergyRatio( + self.sample_rate, + ) + + def calculate_mcd( + self, + wav_targets: torch.Tensor, + wav_predictions: torch.Tensor, + n_mfcc: int = 13, + ) -> torch.Tensor: + """Calculate Mel Cepstral Distortion.""" + mfcc_transform = T.MFCC( + sample_rate=self.sample_rate, + n_mfcc=n_mfcc, + melkwargs={ + "n_fft": 400, + "hop_length": 160, + "n_mels": 23, + "center": False, + }, + ).to(wav_targets.device) + wav_predictions = wav_predictions.to(wav_targets.device) + + ref_mfcc = mfcc_transform(wav_targets) + synth_mfcc = mfcc_transform(wav_predictions) + + mcd = torch.mean( + torch.sqrt( + torch.sum((ref_mfcc - synth_mfcc) ** 2, dim=0), + ), + ) + + return mcd + + def calculate_spectrogram_distance( + self, + wav_targets: torch.Tensor, + wav_predictions: torch.Tensor, + n_fft: int = 2048, + hop_length: int = 512, + ) -> torch.Tensor: + """Calculate spectrogram distance.""" + spec_transform = T.Spectrogram( + n_fft=n_fft, + hop_length=hop_length, + power=None, + ).to(wav_targets.device) + wav_predictions = wav_predictions.to(wav_targets.device) + + # Compute the spectrograms + S1 = spec_transform(wav_targets) + S2 = spec_transform(wav_predictions) + + # Compute the magnitude spectrograms + S1_mag = torch.abs(S1) + S2_mag = torch.abs(S2) + + # Compute the Euclidean distance + dist = torch.dist(S1_mag.flatten(), S2_mag.flatten()) + + return dist + + def calculate_f0_rmse( + self, + wav_targets: torch.Tensor, + wav_predictions: torch.Tensor, + frame_length: int = 2048, + hop_length: int = 512, + ) -> float: + """Calculate F0 RMSE.""" + wav_targets_ = wav_targets.detach().cpu().numpy() + wav_predictions_ = wav_predictions.detach().cpu().numpy() + + # Compute the F0 contour for each audio signal + f0_audio1 = torch.from_numpy( + librosa.yin( + wav_targets_, + fmin=float(librosa.note_to_hz("C2")), + fmax=float(librosa.note_to_hz("C7")), + sr=self.sample_rate, + frame_length=frame_length, + hop_length=hop_length, + ), + ) + f0_audio2 = torch.from_numpy( + librosa.yin( + wav_predictions_, + fmin=float(librosa.note_to_hz("C2")), + fmax=float(librosa.note_to_hz("C7")), + sr=self.sample_rate, + frame_length=frame_length, + hop_length=hop_length, + ), + ) + + # Assuming f0_audio1 and f0_audio2 are PyTorch tensors + rmse = torch.sqrt(torch.mean((f0_audio1 - f0_audio2) ** 2)).item() + + return rmse + + def calculate_jitter_shimmer( + self, + audio: torch.Tensor, + ) -> tuple[float, float]: + r"""Calculate jitter and shimmer of an audio signal. + + Jitter and shimmer are two metrics used in speech signal processing to measure the quality of voice signals. + + Jitter refers to the short-term variability of a signal's fundamental frequency (F0). It is often used as an indicator of voice disorders, as high levels of jitter can indicate a lack of control over the vocal folds. + + Shimmer, on the other hand, refers to the short-term variability in amplitude of the voice signal. Like jitter, high levels of shimmer can be indicative of voice disorders, as they can suggest a lack of control over the vocal tract. + + Summary: + Jitter is the short-term variability of a signal's fundamental frequency (F0). + Shimmer is the short-term variability in amplitude of the voice signal. + + Args: + audio (torch.Tensor): The audio signal to analyze. + + Returns: + tuple[float, float]: The calculated jitter and shimmer values. + """ + # Create a transformation to calculate the spectrogram + spectrogram = T.Spectrogram( + n_fft=self.filter_length * 2, + hop_length=self.hop_length * 2, + power=None, + ) + + spectrogram = spectrogram.to(audio.device) + + # Calculate the spectrogram of the audio signal + amplitude = spectrogram(audio) + + # Calculate the F0 contour using the yin method + f0 = T.Vad(sample_rate=self.sample_rate)(audio) + + # Episilon to avoid division by zero + epsilon = 1e-10 + # Calculate the relative changes in the F0 and amplitude contours + jitter = torch.mean( + torch.abs(torch.diff(f0, dim=-1)) / (torch.diff(f0, dim=-1) + epsilon), + ).item() + shimmer = torch.mean( + torch.abs(torch.diff(amplitude, dim=-1)) + / (torch.diff(amplitude, dim=-1) + epsilon), + ) + + shimmer = torch.abs(shimmer).item() + + return jitter, shimmer + + def wav_metrics(self, wav_predictions: torch.Tensor): + r"""Compute the metrics for the waveforms. + + Args: + wav_predictions (torch.Tensor): The predicted waveforms. + + Returns: + tuple[float, float, float]: The computed metrics. + """ + ermr = self.reverb_modulation_energy_ratio(wav_predictions).item() + jitter, shimmer = self.calculate_jitter_shimmer(wav_predictions) + + return ( + ermr, + jitter, + shimmer, + ) + + def __call__( + self, + wav_predictions: torch.Tensor, + wav_targets: torch.Tensor, + mel_predictions: torch.Tensor, + mel_targets: torch.Tensor, + ) -> MetricsResult: + r"""Compute the metrics. + + Args: + wav_predictions (torch.Tensor): The predicted waveforms. + wav_targets (torch.Tensor): The target waveforms. + mel_predictions (torch.Tensor): The predicted Mel spectrograms. + mel_targets (torch.Tensor): The target Mel spectrograms. + + Returns: + MetricsResult: The computed metrics. + """ + wav_predictions_energy = self.audio_processor.wav_to_energy( + wav_predictions.unsqueeze(0), + self.filter_length, + self.hop_length, + self.win_length, + ) + + wav_targets_energy = self.audio_processor.wav_to_energy( + wav_targets.unsqueeze(0), + self.filter_length, + self.hop_length, + self.win_length, + ) + + energy: torch.Tensor = self.mse_loss(wav_predictions_energy, wav_targets_energy) + + self.si_sdr.to(wav_predictions.device) + self.si_snr.to(wav_predictions.device) + self.c_si_snr.to(wav_predictions.device) + + # New Metrics + si_sdr: torch.Tensor = self.si_sdr(mel_predictions, mel_targets) + si_snr: torch.Tensor = self.si_snr(mel_predictions, mel_targets) + + # New shape: [1, F, T, 2] + mel_predictions_complex = torch.stack( + (mel_predictions, torch.zeros_like(mel_predictions)), + dim=-1, + ) + mel_targets_complex = torch.stack( + (mel_targets, torch.zeros_like(mel_targets)), + dim=-1, + ) + c_si_snr: torch.Tensor = self.c_si_snr( + mel_predictions_complex, + mel_targets_complex, + ) + + mcd = self.calculate_mcd(wav_targets, wav_predictions) + spec_dist = self.calculate_spectrogram_distance(wav_targets, wav_predictions) + f0_rmse = self.calculate_f0_rmse(wav_targets, wav_predictions) + jitter, shimmer = self.calculate_jitter_shimmer(wav_predictions) + + return MetricsResult( + energy, + si_sdr, + si_snr, + c_si_snr, + mcd, + spec_dist, + f0_rmse, + jitter, + shimmer, + ) + + def plot_spectrograms( + self, + mel_target: np.ndarray, + mel_prediction: np.ndarray, + sr: int = 22050, + ): + r"""Plots the mel spectrograms for the target and the prediction.""" + fig, axs = plt.subplots(2, 1, sharex=True, sharey=True, dpi=80) + + img1 = librosa.display.specshow( + mel_target, + x_axis="time", + y_axis="mel", + sr=sr, + ax=axs[0], + ) + axs[0].set_title("Target spectrogram") + fig.colorbar(img1, ax=axs[0], format="%+2.0f dB") + + img2 = librosa.display.specshow( + mel_prediction, + x_axis="time", + y_axis="mel", + sr=sr, + ax=axs[1], + ) + axs[1].set_title("Prediction spectrogram") + fig.colorbar(img2, ax=axs[1], format="%+2.0f dB") + + # Adjust the spacing between subplots + fig.subplots_adjust(hspace=0.5) + + return fig + + def plot_spectrograms_fast( + self, + mel_target: np.ndarray, + mel_prediction: np.ndarray, + sr: int = 22050, + ): + r"""Plots the mel spectrograms for the target and the prediction.""" + fig, axs = plt.subplots(2, 1, sharex=True, sharey=True) + + axs[0].specgram( + mel_target, + aspect="auto", + Fs=sr, + cmap=plt.get_cmap("magma"), # type: ignore + ) + axs[0].set_title("Target spectrogram") + + axs[1].specgram( + mel_prediction, + aspect="auto", + Fs=sr, + cmap=plt.get_cmap("magma"), # type: ignore + ) + axs[1].set_title("Prediction spectrogram") + + # Adjust the spacing between subplots + fig.subplots_adjust(hspace=0.5) + + return fig diff --git a/training/loss/multi_resolution_stft_loss.py b/training/loss/multi_resolution_stft_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..70cab6657d4bb74017ade7854a4707e32ccdcd04 --- /dev/null +++ b/training/loss/multi_resolution_stft_loss.py @@ -0,0 +1,60 @@ +import torch +from torch.nn import Module + +from .stft_loss import STFTLoss + + +class MultiResolutionSTFTLoss(Module): + r"""Multi resolution STFT loss module. + + The Multi resolution STFT loss module is a PyTorch module that computes the spectral convergence and log STFT magnitude losses for a predicted signal and a groundtruth signal at multiple resolutions. The module is designed for speech and audio signal processing tasks, such as speech enhancement and source separation. + + The module takes as input a list of tuples, where each tuple contains the FFT size, hop size, and window length for a particular resolution. For each resolution, the module computes the spectral convergence and log STFT magnitude losses using the STFTLoss module, which is a PyTorch module that computes the STFT of a signal and the corresponding magnitude spectrogram. + + The spectral convergence loss measures the similarity between two magnitude spectrograms, while the log STFT magnitude loss measures the similarity between two logarithmically-scaled magnitude spectrograms. The logarithm is applied to the magnitude spectrograms to convert them to a decibel scale, which is more perceptually meaningful than the linear scale. + + The Multi resolution STFT loss module returns the average spectral convergence and log STFT magnitude losses across all resolutions. This allows the module to capture both fine-grained and coarse-grained spectral information in the predicted and groundtruth signals. + """ + + def __init__( + self, + resolutions: list[tuple[int, int, int]], + ): + r"""Initialize Multi resolution STFT loss module. + + Args: + resolutions (list): List of (FFT size, shift size, window length). + """ + super().__init__() + + self.stft_losses = torch.nn.ModuleList( + [STFTLoss(fs, ss, wl) for fs, ss, wl in resolutions], + ) + + def forward( + self, x: torch.Tensor, y: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + r"""Calculate forward propagation. + + Args: + x (Tensor): Predicted signal (B, T). + y (Tensor): Groundtruth signal (B, T). + + Returns: + Tensor: Multi resolution spectral convergence loss value. + Tensor: Multi resolution log STFT magnitude loss value. + """ + sc_loss = torch.tensor(0.0, device=x.device) + mag_loss = torch.tensor(0.0, device=x.device) + + # Compute the spectral convergence and log STFT magnitude losses for each resolution + for f in self.stft_losses: + sc_l, mag_l = f(x, y) + sc_loss += sc_l + mag_loss += mag_l + + # Average the losses across all resolutions + sc_loss /= len(self.stft_losses) + mag_loss /= len(self.stft_losses) + + return sc_loss, mag_loss diff --git a/training/loss/spectral_convergence_loss.py b/training/loss/spectral_convergence_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..7e4cce0fec9664357a3acd6ecee30b992ab70853 --- /dev/null +++ b/training/loss/spectral_convergence_loss.py @@ -0,0 +1,33 @@ +import torch +from torch.nn import Module + + +class SpectralConvergengeLoss(Module): + r"""Spectral convergence loss module. + Spectral convergence loss is a measure of the similarity between two magnitude spectrograms. + + The spectral convergence loss is calculated as the Frobenius norm of the difference between the predicted and groundtruth magnitude spectrograms, divided by the Frobenius norm of the groundtruth magnitude spectrogram. The Frobenius norm is a matrix norm that is equivalent to the square root of the sum of the squared elements of a matrix. + + The spectral convergence loss is a useful metric for evaluating the quality of a predicted signal, as it measures the degree to which the predicted signal matches the groundtruth signal in terms of its spectral content. A lower spectral convergence loss indicates a better match between the predicted and groundtruth signals. + """ + + def __init__(self): + r"""Initilize spectral convergence loss module.""" + super().__init__() + + def forward(self, x_mag: torch.Tensor, y_mag: torch.Tensor) -> torch.Tensor: + r"""Calculate forward propagation. + + Args: + x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). + y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). + + Returns: + Tensor: Spectral convergence loss value. + """ + # Ensure that x_mag and y_mag have the same size along dimension 1 + min_len = min(x_mag.shape[1], y_mag.shape[1]) + x_mag = x_mag[:, :min_len] + y_mag = y_mag[:, :min_len] + + return torch.norm(y_mag - x_mag, p="fro") / torch.norm(y_mag, p="fro") diff --git a/training/loss/stft.py b/training/loss/stft.py new file mode 100644 index 0000000000000000000000000000000000000000..6a85637fe84150d80a00a1526a9ba959f4440c9a --- /dev/null +++ b/training/loss/stft.py @@ -0,0 +1,31 @@ +import torch + + +def stft( + x: torch.Tensor, + fft_size: int, + hop_size: int, + win_length: int, + window: torch.Tensor, +) -> torch.Tensor: + r"""Perform STFT and convert to magnitude spectrogram. + STFT stands for Short-Time Fourier Transform. It is a signal processing technique that is used to analyze the frequency content of a signal over time. The STFT is computed by dividing a long signal into shorter segments, and then computing the Fourier transform of each segment. This results in a time-frequency representation of the signal, where the frequency content of the signal is shown as a function of time. + + Args: + x (Tensor): Input signal tensor (B, T). + fft_size (int): FFT size. + hop_size (int): Hop size. + win_length (torch.Tensor): Window length. + window (str): Window function type. + + Returns: + Tensor: Magnitude spectrogram (B, #frames, fft_size // 2 + 1). + """ + x_stft = torch.stft(x, fft_size, hop_size, win_length, window, return_complex=True) + x_stft = torch.view_as_real(x_stft) + + real = x_stft[..., 0] + imag = x_stft[..., 1] + + # NOTE (kan-bayashi): clamp is needed to avoid nan or inf + return torch.sqrt(torch.clamp(real**2 + imag**2, min=1e-7)).transpose(2, 1) diff --git a/training/loss/stft_loss.py b/training/loss/stft_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..3f8d0ba7c0e0cbb1e89d8410d1051e0d95808c69 --- /dev/null +++ b/training/loss/stft_loss.py @@ -0,0 +1,61 @@ +import torch +from torch.nn import Module + +from .log_stft_magnitude_loss import LogSTFTMagnitudeLoss +from .spectral_convergence_loss import SpectralConvergengeLoss +from .stft import stft + + +class STFTLoss(Module): + r"""STFT loss module. + + STFT loss is a combination of two loss functions: the spectral convergence loss and the log STFT magnitude loss. + + The spectral convergence loss measures the similarity between two magnitude spectrograms, while the log STFT magnitude loss measures the similarity between two logarithmically-scaled magnitude spectrograms. The logarithm is applied to the magnitude spectrograms to convert them to a decibel scale, which is more perceptually meaningful than the linear scale. + + The STFT loss is a useful metric for evaluating the quality of a predicted signal, as it measures the degree to which the predicted signal matches the groundtruth signal in terms of its spectral content on both a linear and decibel scale. A lower STFT loss indicates a better match between the predicted and groundtruth signals. + + Args: + fft_size (int): FFT size. + shift_size (int): Shift size. + win_length (int): Window length. + """ + + def __init__( + self, + fft_size: int = 1024, + shift_size: int = 120, + win_length: int = 600, + ): + r"""Initialize STFT loss module.""" + super().__init__() + + self.fft_size = fft_size + self.shift_size = shift_size + self.win_length = win_length + + self.register_buffer("window", torch.hann_window(win_length)) + + self.spectral_convergenge_loss = SpectralConvergengeLoss() + self.log_stft_magnitude_loss = LogSTFTMagnitudeLoss() + + def forward( + self, x: torch.Tensor, y: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + r"""Calculate forward propagation. + + Args: + x (Tensor): Predicted signal (B, T). + y (Tensor): Groundtruth signal (B, T). + + Returns: + Tensor: Spectral convergence loss value. + Tensor: Log STFT magnitude loss value. + """ + x_mag = stft(x, self.fft_size, self.shift_size, self.win_length, self.window) + y_mag = stft(y, self.fft_size, self.shift_size, self.win_length, self.window) + + sc_loss = self.spectral_convergenge_loss(x_mag, y_mag) + mag_loss = self.log_stft_magnitude_loss(x_mag, y_mag) + + return sc_loss, mag_loss diff --git a/training/loss/stft_magnitude_loss.py b/training/loss/stft_magnitude_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..1115555f53248e6d03c845b01e47e4ba75ec80e5 --- /dev/null +++ b/training/loss/stft_magnitude_loss.py @@ -0,0 +1,50 @@ +import torch + + +class STFTMagnitudeLoss(torch.nn.Module): + """STFT magnitude loss module. + + See [Arik et al., 2018](https://arxiv.org/abs/1808.06719) + and [Engel et al., 2020](https://arxiv.org/abs/2001.04643v1) + + Args: + log (bool, optional): Log-scale the STFT magnitudes, + or use linear scale. Default: True + distance (str, optional): Distance function ["L1", "L2"]. Default: "L1" + reduction (str, optional): Reduction of the loss elements. Default: "mean" + """ + + def __init__( + self, + log: bool = True, + distance: str = "L1", + reduction: str = "mean", + epsilon: float = 1e-8, + ): + super().__init__() + + self.log = log + self.epsilon = epsilon + + if distance == "L1": + self.distance = torch.nn.L1Loss(reduction=reduction) + elif distance == "L2": + self.distance = torch.nn.MSELoss(reduction=reduction) + else: + raise ValueError(f"Invalid distance: '{distance}'.") + + def forward(self, x_mag: torch.Tensor, y_mag: torch.Tensor) -> torch.Tensor: + r"""Calculate forward propagation. + + Args: + x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). + y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). + + Returns: + Tensor: Log STFT magnitude loss value. + """ + if self.log: + x_mag = torch.sign(x_mag) * torch.log(torch.abs(x_mag + self.epsilon)) + y_mag = torch.sign(y_mag) * torch.log(torch.abs(y_mag + self.epsilon)) + + return self.distance(x_mag, y_mag) diff --git a/training/loss/tests/__init__.py b/training/loss/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/training/loss/tests/test_bin_loss.py b/training/loss/tests/test_bin_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..53bc25305c776962529cb174fd9bda1c1f456997 --- /dev/null +++ b/training/loss/tests/test_bin_loss.py @@ -0,0 +1,49 @@ +import unittest + +import torch + +from training.loss import BinLoss + + +class TestBinLoss(unittest.TestCase): + def setUp(self): + self.bin_loss = BinLoss() + + def test_forward_hard_attention(self): + # Test with hard attention + hard_attention = torch.tensor([1, 0, 1, 0]) + soft_attention = torch.tensor([0.9, 0.1, 0.8, 0.2]) + + loss = self.bin_loss(hard_attention, soft_attention) + + expected_loss = -(torch.log(torch.tensor([0.9, 0.8]))).sum() / 2 + + self.assertAlmostEqual(loss.item(), expected_loss.item()) + + def test_forward_soft_attention(self): + # Test with soft attention + hard_attention = torch.tensor([1, 0, 1, 0]) + soft_attention = torch.tensor([0.9, 0.1, 0.8, 0.2], requires_grad=True) + + loss = self.bin_loss(hard_attention, soft_attention) + expected_loss = ( + -(torch.log(torch.tensor([0.9, 0.8], requires_grad=True))).sum() / 2 + ) + expected_loss.backward() + loss.backward() + + self.assertAlmostEqual(loss.item(), expected_loss.item()) + self.assertIsNotNone(soft_attention.grad) + + if soft_attention.grad is not None: + self.assertTrue( + torch.allclose( + soft_attention.grad, + torch.tensor([-0.5556, 0.0000, -0.6250, 0.0000]), + atol=1e-4, + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_delightful_tts_loss.py b/training/loss/tests/test_delightful_tts_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..d57a621c2f1d0216b5f99ef3e4f6a69e1ad82158 --- /dev/null +++ b/training/loss/tests/test_delightful_tts_loss.py @@ -0,0 +1,73 @@ +import unittest + +import torch + +from models.config import AcousticENModelConfig +from training.loss.delightful_tts_loss import ( + DelightfulTTSLoss, + ForwardSumLoss, + sample_wise_min_max, + sequence_mask, +) + + +class TestLosses(unittest.TestCase): + def setUp(self): + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + def test_sequence_mask(self): + sequence_length = torch.tensor([2, 3, 4], device=self.device) + max_len = 5 + mask = sequence_mask(sequence_length, max_len) + expected_mask = torch.tensor([[True, True, False, False, False], + [True, True, True, False, False], + [True, True, True, True, False]], device=self.device) + self.assertTrue(torch.equal(mask, expected_mask)) + + def test_sample_wise_min_max(self): + x = torch.tensor([[[1, 1], [1, 0], [0, 1]]], dtype=torch.float32, device=self.device) + normalized_x = sample_wise_min_max(x) + expected_normalized_x = torch.tensor([[ + [1., 1.], + [1., 0.], + [0., 1.], + ]], device=self.device) + self.assertTrue(torch.allclose(normalized_x, expected_normalized_x)) + + def test_ForwardSumLoss(self): + loss_function = ForwardSumLoss() + attn_logprob = torch.randn((1, 1, 11, 11)) + src_lens = torch.ones((1,), dtype=torch.long) + mel_lens = torch.ones((1,), dtype=torch.long) + loss = loss_function(attn_logprob, src_lens, mel_lens) + self.assertTrue(isinstance(loss, torch.Tensor)) + + def test_DelightfulTTSLoss(self): + model_config = AcousticENModelConfig() + loss_function = DelightfulTTSLoss(model_config) + mel_output = torch.randn((1, 11, 11)) + mel_target = torch.randn((1, 11, 11)) + mel_lens = torch.ones((1,), dtype=torch.long) + dur_output = torch.randn((1, 11)) + dur_target = torch.randn((1, 11)) + pitch_output = torch.randn((1, 11)) + pitch_target = torch.randn((1, 11)) + energy_output = torch.randn((1, 11)) + energy_target = torch.randn((1, 11)) + src_lens = torch.ones((1,), dtype=torch.long) + p_prosody_ref = torch.randn((1, 11, 11)) + p_prosody_pred = torch.randn((1, 11, 11)) + u_prosody_ref = torch.randn((1, 11, 11)) + u_prosody_pred = torch.randn((1, 11, 11)) + aligner_logprob = torch.randn((1, 1, 11, 11)) + aligner_hard = torch.randn((1, 11, 11)) + aligner_soft = torch.randn((1, 11, 11)) + total_loss, _, _, _, _, _, _, _, _, _ = loss_function( + mel_output, mel_target, mel_lens, dur_output, dur_target, pitch_output, pitch_target, + energy_output, energy_target, src_lens, p_prosody_ref, p_prosody_pred, + u_prosody_ref, u_prosody_pred, aligner_logprob, aligner_hard, aligner_soft, + ) + self.assertTrue(isinstance(total_loss, torch.Tensor)) + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_fast_speech_2_loss_gen.py b/training/loss/tests/test_fast_speech_2_loss_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..c8864c7e7adb30067bc7d5162e6b357cf06b9231 --- /dev/null +++ b/training/loss/tests/test_fast_speech_2_loss_gen.py @@ -0,0 +1,114 @@ +import unittest + +import torch + +from training.loss.fast_speech_2_loss_gen import FastSpeech2LossGen + + +class TestFastSpeech2LossGen(unittest.TestCase): + def setUp(self): + self.loss_gen = FastSpeech2LossGen() + + def test_forward(self): + # Reproducible results + torch.random.manual_seed(0) + + # Test with all inputs of shape (1, 11) + src_masks = torch.zeros((1, 11), dtype=torch.bool) + mel_masks = torch.zeros((1, 11), dtype=torch.bool) + mel_targets = torch.randn((1, 11, 11)) + # postnet = torch.randn((1, 11, 11)) + mel_predictions = torch.randn((1, 11, 11)) + log_duration_predictions = torch.randn((1, 11)) + u_prosody_ref = torch.randn((1, 11)) + u_prosody_pred = torch.randn((1, 11)) + p_prosody_ref = torch.randn((1, 11, 11)) + p_prosody_pred = torch.randn((1, 11, 11)) + durations = torch.randn((1, 11)) + pitch_predictions = torch.randn((1, 11)) + p_targets = torch.randn((1, 11)) + attn_logprob = torch.randn((1, 1, 11, 11)) + attn_soft = torch.randn((1, 11, 11)) + attn_hard = torch.randn((1, 11, 11)) + step = 20000 + src_lens = torch.ones((1,), dtype=torch.long) + mel_lens = torch.ones((1,), dtype=torch.long) + energy_pred = torch.randn((1, 11)) + energy_target = torch.randn((1, 11)) + + ( + total_loss, + mel_loss, + # mel_loss_postnet, + ssim_loss, + # ssim_loss_postnet, + duration_loss, + u_prosody_loss, + p_prosody_loss, + pitch_loss, + ctc_loss, + bin_loss, + energy_loss, + ) = self.loss_gen.forward( + src_masks, + mel_masks, + mel_targets, + mel_predictions, + # postnet, + log_duration_predictions, + u_prosody_ref, + u_prosody_pred, + p_prosody_ref, + p_prosody_pred, + durations, + pitch_predictions, + p_targets, + attn_logprob, + attn_soft, + attn_hard, + step, + src_lens, + mel_lens, + energy_pred, + energy_target, + ) + + self.assertIsInstance(total_loss, torch.Tensor) + self.assertIsInstance(mel_loss, torch.Tensor) + # self.assertIsInstance(mel_loss_postnet, torch.Tensor) + self.assertIsInstance(ssim_loss, torch.Tensor) + # self.assertIsInstance(ssim_loss_postnet, torch.Tensor) + self.assertIsInstance(duration_loss, torch.Tensor) + self.assertIsInstance(u_prosody_loss, torch.Tensor) + self.assertIsInstance(p_prosody_loss, torch.Tensor) + self.assertIsInstance(pitch_loss, torch.Tensor) + self.assertIsInstance(ctc_loss, torch.Tensor) + self.assertIsInstance(bin_loss, torch.Tensor) + self.assertIsInstance(energy_loss, torch.Tensor) + + # Assert the value of losses + self.assertTrue( + torch.all( + torch.tensor( + [ + total_loss, + mel_loss, + # mel_loss_postnet, + ssim_loss, + # ssim_loss_postnet, + duration_loss, + u_prosody_loss, + p_prosody_loss, + pitch_loss, + ctc_loss, + bin_loss, + energy_loss, + ], + ) + >= 0, + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_forward_sum_loss.py b/training/loss/tests/test_forward_sum_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..e46f18f8c0a440d474a217e7ae0cd7c0cbded9e5 --- /dev/null +++ b/training/loss/tests/test_forward_sum_loss.py @@ -0,0 +1,33 @@ +import unittest + +import torch + +from training.loss import ForwardSumLoss + + +class TestForwardSumLoss(unittest.TestCase): + def setUp(self): + self.forward_sum_loss = ForwardSumLoss() + + def test_forward(self): + # Reproducible results + torch.random.manual_seed(0) + + T = 2 # Input sequence length + C = 2 # Number of classes (including blank) + N = 1 # Batch size + S = 1 # Target sequence length of longest target in batch (padding length) + + attn_logprob = torch.randn(T, N, C, C).log_softmax(2).detach().requires_grad_() + + in_lens = torch.full(size=(T,), fill_value=T, dtype=torch.long) + out_lens = torch.randint(low=S, high=T, size=(T,), dtype=torch.long) + + loss = self.forward_sum_loss(attn_logprob, in_lens, out_lens) + expected_loss = torch.tensor([0.0]) + + self.assertTrue(torch.allclose(loss, expected_loss)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_log_stft_magnitude_loss.py b/training/loss/tests/test_log_stft_magnitude_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..a7c0971dab6c6e71820cced4e092a0f57b4d99bb --- /dev/null +++ b/training/loss/tests/test_log_stft_magnitude_loss.py @@ -0,0 +1,38 @@ +import unittest + +import torch + +from training.loss import LogSTFTMagnitudeLoss + + +class TestLogSTFTMagnitudeLoss(unittest.TestCase): + def test_log_stft_magnitude_loss(self): + # Test the log STFT magnitude loss function with random input tensors + loss_fn = LogSTFTMagnitudeLoss() + + x_mag = torch.randn(4, 100, 513) + y_mag = torch.randn(4, 100, 513) + + loss = loss_fn(x_mag, y_mag) + + self.assertIsInstance(loss, torch.Tensor) + self.assertEqual(loss.shape, torch.Size([])) + + def test_log_stft_magnitude_loss_nonzero(self): + # Test the log STFT magnitude loss function with non-zero loss + loss_fn = LogSTFTMagnitudeLoss() + + x_mag = torch.tensor([[1, 4, 9, 64], [1, 1, 1, 2]]) + y_mag = torch.tensor([[1, 8, 16, 256], [1, 1, 2, 2]]) + + loss = loss_fn(x_mag, y_mag) + + self.assertIsInstance(loss, torch.Tensor) + self.assertEqual(loss.shape, torch.Size([])) + + expected = torch.tensor(0.4185) + self.assertTrue(torch.allclose(loss, expected, rtol=1e-4, atol=1e-4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_metrics.py b/training/loss/tests/test_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..cbefdda2078774af759e0ec03f098c0366dc4410 --- /dev/null +++ b/training/loss/tests/test_metrics.py @@ -0,0 +1,53 @@ +import unittest + +import torch + +from training.loss.metrics import Metrics + + +class TestMetrics(unittest.TestCase): + def setUp(self): + self.metrics = Metrics() + + # Set the frequency of the pitch (in Hz) + self.pitch_freq = 440.0 + self.duration = 1.0 + self.sr = 22050 + + # Generate a time vector for the audio signal + self.t = torch.linspace(0, self.duration, int(self.sr * self.duration)) + + # Generate a sinusoidal waveform with the specified pitch frequency + self.audio = torch.sin(2 * torch.pi * self.pitch_freq * self.t).unsqueeze(0) + + def test_calculate_mcd(self): + wav_targets = torch.randn(1, 22050) + wav_predictions = torch.randn(1, 22050) + mcd = self.metrics.calculate_mcd(wav_targets, wav_predictions) + self.assertIsInstance(mcd, torch.Tensor) + + def test_calculate_spectrogram_distance(self): + wav_targets = torch.randn(1, 22050) + wav_predictions = torch.randn(1, 22050) + dist = self.metrics.calculate_spectrogram_distance(wav_targets, wav_predictions) + self.assertIsInstance(dist, torch.Tensor) + + def test_calculate_f0_rmse(self): + wav_targets = torch.randn(1, 22050) + wav_predictions = torch.randn(1, 22050) + rmse = self.metrics.calculate_f0_rmse(wav_targets, wav_predictions) + self.assertIsInstance(rmse, float) + + def test_calculate_jitter_shimmer(self): + jitter, shimmer = self.metrics.calculate_jitter_shimmer(self.audio) + self.assertIsInstance(jitter, float) + self.assertIsInstance(shimmer, float) + + def test_wav_metrics(self): + ermr, jitter, shimmer = self.metrics.wav_metrics(self.audio) + self.assertIsInstance(ermr, float) + self.assertIsInstance(jitter, float) + self.assertIsInstance(shimmer, float) + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_multi_resolution_stft_loss.py b/training/loss/tests/test_multi_resolution_stft_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..1e0498258caa10cc88b6004aa2bbcd6e49f42201 --- /dev/null +++ b/training/loss/tests/test_multi_resolution_stft_loss.py @@ -0,0 +1,66 @@ +import unittest + +import torch + +from training.loss import MultiResolutionSTFTLoss + + +class TestMultiResolutionSTFTLoss(unittest.TestCase): + def setUp(self): + torch.random.manual_seed(0) + self.loss_fn = MultiResolutionSTFTLoss([(1024, 120, 600), (2048, 240, 1200)]) + + self.x = torch.randn( + 4, + 16000, + ) + self.y = torch.randn( + 4, + 16000, + ) + + def test_multi_resolution_stft_loss(self): + # Test the MultiResolutionSTFTLoss class with random input tensors + + sc_loss, mag_loss = self.loss_fn(self.x, self.y) + + self.assertIsInstance(sc_loss, torch.Tensor) + self.assertEqual(sc_loss.shape, torch.Size([])) + self.assertIsInstance(mag_loss, torch.Tensor) + self.assertEqual(mag_loss.shape, torch.Size([])) + + def test_multi_resolution_stft_loss_nonzero(self): + # Test the MultiResolutionSTFTLoss class with input tensors that have a non-zero loss value + torch.manual_seed(0) + + x = torch.randn( + 4, + 16000, + ) + y = torch.randn( + 4, + 16000, + ) + + sc_loss, mag_loss = self.loss_fn(x, y) + + self.assertIsInstance(sc_loss, torch.Tensor) + self.assertEqual(sc_loss.shape, torch.Size([])) + self.assertIsInstance(mag_loss, torch.Tensor) + self.assertEqual(mag_loss.shape, torch.Size([])) + + expected_sc_loss = torch.tensor( + 0.6571, + ) + self.assertTrue(torch.allclose(sc_loss, expected_sc_loss, atol=1e-4)) + + expected_mag_loss = torch.tensor( + 0.7007, + ) + self.assertTrue( + torch.allclose(mag_loss, expected_mag_loss, rtol=1e-4, atol=1e-4), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_sample_wise_min_max.py b/training/loss/tests/test_sample_wise_min_max.py new file mode 100644 index 0000000000000000000000000000000000000000..40f9b96e231606ed18063c989cbfcd51f8709b41 --- /dev/null +++ b/training/loss/tests/test_sample_wise_min_max.py @@ -0,0 +1,54 @@ +import unittest + +import torch + +from training.loss.utils import sample_wise_min_max + + +class TestSampleWiseMinMax(unittest.TestCase): + def test_sample_wise_min_max(self): + # Test case 1: Test with batch size 1 + x = torch.tensor([[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]) + normalized = sample_wise_min_max(x) + expected = torch.tensor( + [[[0.0, 0.125, 0.25], [0.375, 0.5, 0.625], [0.75, 0.875, 1.0]]], + ) + + self.assertTrue(torch.allclose(normalized, expected)) + + # Test case 2: Test with batch size 2 + x = torch.tensor( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[2, 4, 6], [8, 10, 12], [14, 16, 18]], + ], + ) + normalized = sample_wise_min_max(x) + expected = torch.tensor( + [ + [[0.0, 0.125, 0.25], [0.375, 0.5, 0.625], [0.75, 0.875, 1.0]], + [[0.0, 0.125, 0.25], [0.375, 0.5, 0.625], [0.75, 0.875, 1.0]], + ], + ) + + self.assertTrue(torch.allclose(normalized, expected)) + + # Test case 3: Test with negative values + x = torch.tensor( + [ + [[-1, -2, -3], [-4, -5, -6], [-7, -8, -9]], + [[-2, -4, -6], [-8, -10, -12], [-14, -16, -18]], + ], + ) + normalized = sample_wise_min_max(x) + expected = torch.tensor( + [ + [[1.0, 0.875, 0.75], [0.625, 0.5, 0.375], [0.25, 0.125, 0.0]], + [[1.0, 0.875, 0.75], [0.625, 0.5, 0.375], [0.25, 0.125, 0.0]], + ], + ) + self.assertTrue(torch.allclose(normalized, expected)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_spectral_convergence_loss.py b/training/loss/tests/test_spectral_convergence_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..73f6b77c358d0f7dd96ac002317217e8db036abf --- /dev/null +++ b/training/loss/tests/test_spectral_convergence_loss.py @@ -0,0 +1,39 @@ +import unittest + +import torch + +from training.loss import SpectralConvergengeLoss + + +class TestSpectralConvergengeLoss(unittest.TestCase): + def test_spectral_convergence_loss(self): + # Test the spectral convergence loss function with random input tensors + loss_fn = SpectralConvergengeLoss() + + x_mag = torch.randn(4, 100, 513) + y_mag = torch.randn(4, 100, 513) * 0.1 + + loss = loss_fn(x_mag, y_mag) + + self.assertIsInstance(loss, torch.Tensor) + self.assertEqual(loss.shape, torch.Size([])) + self.assertGreater(loss, 0.0) + + def test_spectral_convergence_small_vectors(self): + # Test the spectral convergence loss function with non-zero loss + loss_fn = SpectralConvergengeLoss() + + x_mag = torch.tensor([[1, 4, 9, 64], [1, 1, 1, 2]], dtype=torch.float32) + y_mag = torch.tensor([[1, 8, 16, 256], [1, 1, 2, 2]], dtype=torch.float32) + + loss = loss_fn(x_mag, y_mag) + + self.assertIsInstance(loss, torch.Tensor) + self.assertEqual(loss.shape, torch.Size([])) + + expected = torch.tensor(0.7488) + self.assertTrue(torch.allclose(loss, expected, rtol=1e-4, atol=1e-4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_stft.py b/training/loss/tests/test_stft.py new file mode 100644 index 0000000000000000000000000000000000000000..c884da22fbec11d6e73e6f4f8c997beb9ebbdc91 --- /dev/null +++ b/training/loss/tests/test_stft.py @@ -0,0 +1,25 @@ +import unittest + +import torch + +from training.loss.stft import stft + + +class TestSTFT(unittest.TestCase): + def test_stft(self): + # Test the STFT function with a random input signal + x = torch.randn(4, 16384) + fft_size = 1024 + hop_size = 256 + win_length = 1024 + window = torch.hann_window(win_length) + output = stft(x, fft_size, hop_size, win_length, window) + self.assertEqual(output.shape[0], 4) + self.assertEqual(output.shape[2], fft_size // 2 + 1) + self.assertEqual( + output.shape[1], (16384 - win_length) // hop_size + x.shape[0] + 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_stft_loss.py b/training/loss/tests/test_stft_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..7605a82773d4df9da788c2c3ac480e5a36c00d75 --- /dev/null +++ b/training/loss/tests/test_stft_loss.py @@ -0,0 +1,59 @@ +import unittest + +import torch + +from training.loss import STFTLoss + + +class TestSTFTLoss(unittest.TestCase): + def test_stft_loss(self): + torch.random.manual_seed(0) + # Test the STFT loss function with random input tensors + loss_fn = STFTLoss() + + x = torch.randn( + 4, + 16000, + ) + y = torch.randn( + 4, + 16000, + ) + + sc_loss, mag_loss = loss_fn(x, y) + + self.assertIsInstance(sc_loss, torch.Tensor) + self.assertEqual(sc_loss.shape, torch.Size([])) + self.assertIsInstance(mag_loss, torch.Tensor) + self.assertEqual(mag_loss.shape, torch.Size([])) + + def test_stft_loss_nonzero(self): + # Test the STFT loss function with non-zero loss + loss_fn = STFTLoss() + + # Reproducibility + torch.manual_seed(0) + + x_mag = torch.randn(4, 16000, dtype=torch.float32) + y_mag = torch.randn(4, 16000, dtype=torch.float32) + + sc_loss, mag_loss = loss_fn(x_mag, y_mag) + + self.assertIsInstance(sc_loss, torch.Tensor) + self.assertEqual(sc_loss.shape, torch.Size([])) + + self.assertIsInstance(mag_loss, torch.Tensor) + self.assertEqual(mag_loss.shape, torch.Size([])) + + self.assertGreater(sc_loss, 0.0) + self.assertGreater(mag_loss, 0.0) + + expected_sc = torch.tensor(0.6559) + self.assertTrue(torch.allclose(sc_loss, expected_sc, rtol=1e-4, atol=1e-4)) + + expected_mag = torch.tensor(0.6977) + self.assertTrue(torch.allclose(mag_loss, expected_mag, rtol=1e-4, atol=1e-4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/tests/test_univnet_loss.py b/training/loss/tests/test_univnet_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..7b8b49f0e2af51ddf91034818a8b9789e40d7ad2 --- /dev/null +++ b/training/loss/tests/test_univnet_loss.py @@ -0,0 +1,72 @@ +import unittest + +import torch + +from training.loss import UnivnetLoss + + +class TestUnivnetLoss(unittest.TestCase): + def setUp(self): + torch.random.manual_seed(42) + self.loss_module = UnivnetLoss() + + def test_forward(self): + # Create some fake input data + audio = torch.randn(1, 1, 22050) + fake_audio = torch.randn(1, 1, 22050) + res_fake = [(torch.randn(1, 1, 22050), torch.randn(1))] + period_fake = [(torch.randn(1, 1, 22050), torch.randn(1))] + res_real = [(torch.randn(1, 1, 22050), torch.randn(1))] + period_real = [(torch.randn(1, 1, 22050), torch.randn(1))] + + # Call the forward method + output = self.loss_module.forward( + audio, + fake_audio, + res_fake, + period_fake, + res_real, + period_real, + ) + + # Check that the output is a tuple with the expected lens + self.assertIsInstance(output, tuple) + + self.assertEqual(len(output), 6) + + ( + total_loss_gen, + total_loss_disc, + stft_loss, + score_loss, + esr_loss, + snr_loss, + ) = output + + self.assertIsInstance(total_loss_gen, torch.Tensor) + self.assertIsInstance(total_loss_disc, torch.Tensor) + self.assertIsInstance(stft_loss, torch.Tensor) + self.assertIsInstance(score_loss, torch.Tensor) + self.assertIsInstance(esr_loss, torch.Tensor) + self.assertIsInstance(snr_loss, torch.Tensor) + + # Assert the value of losses + self.assertTrue( + torch.all( + torch.tensor( + [ + total_loss_gen, + total_loss_disc, + stft_loss, + score_loss, + esr_loss, + snr_loss, + ], + ) + >= 0, + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/loss/torch_stft.py b/training/loss/torch_stft.py new file mode 100644 index 0000000000000000000000000000000000000000..955595cef6d43e83fb506ccfb84bfd51b932bca7 --- /dev/null +++ b/training/loss/torch_stft.py @@ -0,0 +1,269 @@ +from typing import Literal, Optional, Tuple, Union + +import librosa +import torch +from torch import Tensor, nn +from torch.nn import functional + + +class TorchSTFT(nn.Module): + r"""Some of the audio processing funtions using Torch for faster batch processing. + + Args: + n_fft (int): FFT window size for STFT. + hop_length (int): number of frames between STFT columns. + win_length (int, optional): STFT window length. + pad_wav (bool, optional): If True pad the audio with (n_fft - hop_length) / 2). Defaults to False. + window (str, optional): The name of a function to create a window tensor that is applied/multiplied to each frame/window. Defaults to "hann_window" + sample_rate (int, optional): target audio sampling rate. Defaults to None. + mel_fmin (int, optional): minimum filter frequency for computing melspectrograms. Defaults to None. + mel_fmax (int, optional): maximum filter frequency for computing melspectrograms. Defaults to None. + n_mels (int, optional): number of melspectrogram dimensions. Defaults to None. + use_mel (bool, optional): If True compute the melspectrograms otherwise. Defaults to False. + do_amp_to_db_linear (bool, optional): enable/disable amplitude to dB conversion of linear spectrograms. Defaults to False. + spec_gain (float, optional): gain applied when converting amplitude to DB. Defaults to 1.0. + power (float, optional): Exponent for the magnitude spectrogram, e.g., 1 for energy, 2 for power, etc. Defaults to None. + use_htk (bool, optional): Use HTK formula in mel filter instead of Slaney. + mel_norm (None, 'slaney', or number, optional): If 'slaney', divide the triangular mel weights by the width of the mel band (area normalization). + + If numeric, use `librosa.util.normalize` to normalize each filter by to unit l_p norm. + See `librosa.util.normalize` for a full description of supported norm values (including `+-np.inf`). + Otherwise, leave all the triangles aiming for a peak value of 1.0. Defaults to "slaney". + """ + + def __init__( + self, + n_fft: int, + hop_length: int, + win_length: int, + pad_wav: bool = False, + window: str = "hann_window", + sample_rate: int = 22050, + mel_fmin: int = 0, + mel_fmax: Optional[int] = None, + n_mels: int = 80, + use_mel: bool = False, + do_amp_to_db:bool = False, + spec_gain: float = 1.0, + power: Optional[float] = None, + use_htk: bool = False, + mel_norm: Optional[Union[Literal["slaney"], float]] = "slaney", + normalized: bool = False, + ): + super().__init__() + self.n_fft = n_fft + self.hop_length = hop_length + self.win_length = win_length + self.pad_wav = pad_wav + self.sample_rate = sample_rate + self.mel_fmin = mel_fmin + self.mel_fmax = mel_fmax + self.n_mels = n_mels + self.use_mel = use_mel + self.do_amp_to_db = do_amp_to_db + self.spec_gain = spec_gain + self.power = power + self.use_htk = use_htk + self.window = nn.Parameter(getattr(torch, window)(win_length), requires_grad=False) + self.normalized = normalized + + self.mel_norm: Optional[Union[Literal["slaney"], float]] = mel_norm + self.mel_basis = None + + if use_mel: + self._build_mel_basis() + + def __call__(self, x: Tensor): + """Compute spectrogram frames by torch based stft. + + Args: + x (Tensor): input waveform + + Returns: + Tensor: spectrogram frames. + + Shapes: + x: [B x T] or [:math:`[B, 1, T]`] + """ + if x.ndim == 2: + x = x.unsqueeze(1) + + if self.pad_wav: + padding = int((self.n_fft - self.hop_length) / 2) + x = torch.nn.functional.pad(x, (padding, padding), mode="reflect") + + # B x D x T x 2 + o = torch.stft( + x.squeeze(1), + self.n_fft, + self.hop_length, + self.win_length, + self.window, + center=True, + pad_mode="reflect", # compatible with audio.py + normalized=self.normalized, + onesided=True, + return_complex=False, + ) + + M = o[:, :, :, 0] + P = o[:, :, :, 1] + + S = torch.sqrt(torch.clamp(M**2 + P**2, min=1e-8)) + + if self.power is not None: + S = S**self.power + + if self.use_mel and self.mel_basis is not None: + S = torch.matmul(self.mel_basis.to(x), S) + + if self.do_amp_to_db: + S = self._amp_to_db(S, spec_gain=self.spec_gain) + + return S + + def _build_mel_basis(self): + r"""Builds the mel basis for the spectrogram transformation. + This method is called during initialization if use_mel is set to True. + """ + mel_basis = librosa.filters.mel( + sr=self.sample_rate, + n_fft=self.n_fft, + n_mels=self.n_mels, + fmin=self.mel_fmin, + fmax=self.mel_fmax, + htk=self.use_htk, + norm=self.mel_norm, + ) + self.mel_basis = torch.from_numpy(mel_basis).float() + + @staticmethod + def _amp_to_db(x: Tensor, spec_gain: float = 1.0) -> Tensor: + r"""Converts amplitude to decibels. + + Args: + x (Tensor): The amplitude tensor to convert. + spec_gain (float, optional): The gain applied when converting. Defaults to 1.0. + + Returns: + Tensor: The converted tensor in decibels. + """ + return torch.log(torch.clamp(x, min=1e-5) * spec_gain) + + @staticmethod + def _db_to_amp(x: Tensor, spec_gain: float = 1.0) -> Tensor: + r"""Converts decibels to amplitude. + + Args: + x (Tensor): The decibel tensor to convert. + spec_gain (float, optional): The gain applied when converting. Defaults to 1.0. + + Returns: + Tensor: The converted tensor in amplitude. + """ + return torch.exp(x) / spec_gain + + +class STFTLoss(nn.Module): + r"""STFT loss. Input generate and real waveforms are converted + to spectrograms compared with L1 and Spectral convergence losses. + It is from ParallelWaveGAN paper https://arxiv.org/pdf/1910.11480.pdf + + Attributes: + n_fft (int): The FFT size. + hop_length (int): The hop (stride) size. + win_length (int): The window size. + stft (TorchSTFT): The STFT function. + + Methods: + forward(y_hat: Tensor, y: Tensor) + Compute the STFT loss. + """ + + def __init__(self, n_fft: int, hop_length: int, win_length: int): + r"""Constructs all the necessary attributes for the STFTLoss object. + + Args: + n_fft (int): The FFT size. + hop_length (int): The hop (stride) size. + win_length (int): The window size. + """ + super().__init__() + self.n_fft = n_fft + self.hop_length = hop_length + self.win_length = win_length + self.stft = TorchSTFT(n_fft, hop_length, win_length) + + def forward(self, y_hat: Tensor, y: Tensor): + r"""Compute the STFT loss. + + Args: + y_hat (Tensor): The generated waveforms. + y (Tensor): The real waveforms. + + Returns: + loss_mag (Tensor): The magnitude loss. + loss_sc (Tensor): The spectral convergence loss. + """ + y_hat_M = self.stft(y_hat) + y_M = self.stft(y) + + # magnitude loss + loss_mag = functional.l1_loss(torch.log(y_M), torch.log(y_hat_M)) + + # spectral convergence loss + loss_sc = torch.norm(y_M - y_hat_M, p="fro") / torch.norm(y_M, p="fro") + return loss_mag, loss_sc + + +class MultiScaleSTFTLoss(nn.Module): + """Multi-scale STFT loss. Input generate and real waveforms are converted + to spectrograms compared with L1 and Spectral convergence losses. + It is from ParallelWaveGAN paper https://arxiv.org/pdf/1910.11480.pdf + + Attributes: + loss_funcs (torch.nn.ModuleList): A list of STFTLoss modules for different scales. + + Methods: + forward(y_hat: Tensor, y: Tensor) -> Tuple[Tensor, Tensor] + Compute the multi-scale STFT loss. + """ + + def __init__( + self, + n_ffts: Tuple[int, int, int] = (1024, 2048, 512), + hop_lengths: Tuple[int, int, int] = (120, 240, 50), + win_lengths: Tuple[int, int, int] = (600, 1200, 240), + ): + r"""Initialize the MultiScaleSTFTLoss module. + + Args: + n_ffts (Tuple[int, int, int], optional): The FFT sizes for the STFTLoss modules. Defaults to (1024, 2048, 512). + hop_lengths (Tuple[int, int, int], optional): The hop lengths for the STFTLoss modules. Defaults to (120, 240, 50). + win_lengths (Tuple[int, int, int], optional): The window lengths for the STFTLoss modules. Defaults to (600, 1200, 240). + """ + super().__init__() + self.loss_funcs = torch.nn.ModuleList() + for n_fft, hop_length, win_length in zip(n_ffts, hop_lengths, win_lengths): + self.loss_funcs.append(STFTLoss(n_fft, hop_length, win_length)) + + def forward(self, y_hat: Tensor, y: Tensor): + r"""Compute the multi-scale STFT loss. + + Args: + y_hat (Tensor): The generated waveforms. + y (Tensor): The real waveforms. + + Returns: + Tuple[Tensor, Tensor]: The magnitude and spectral convergence losses. + """ + N = len(self.loss_funcs) + loss_sc = 0 + loss_mag = 0 + for f in self.loss_funcs: + lm, lsc = f(y_hat, y) + loss_mag += lm + loss_sc += lsc + loss_sc /= N + loss_mag /= N + return loss_mag, loss_sc diff --git a/training/loss/univnet_loss.py b/training/loss/univnet_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..d0804d10e84c464114ae516f66f41880632fdb00 --- /dev/null +++ b/training/loss/univnet_loss.py @@ -0,0 +1,99 @@ +from typing import List, Tuple + +from auraloss.time import ESRLoss, SDSDRLoss, SISDRLoss, SNRLoss +import torch +from torch import Tensor +from torch.nn import Module + +from models.config import VocoderBasicConfig, VocoderModelConfig + +from .multi_resolution_stft_loss import MultiResolutionSTFTLoss + + +class UnivnetLoss(Module): + r"""UnivnetLoss is a PyTorch Module that calculates the generator and discriminator losses for Univnet.""" + + def __init__(self): + r"""Initializes the UnivnetLoss module.""" + super().__init__() + + train_config = VocoderBasicConfig() + + self.stft_lamb = train_config.stft_lamb + self.model_config = VocoderModelConfig() + + self.stft_criterion = MultiResolutionSTFTLoss(self.model_config.mrd.resolutions) + self.esr_loss = ESRLoss() + self.sisdr_loss = SISDRLoss() + self.snr_loss = SNRLoss() + self.sdsdr_loss = SDSDRLoss() + + def forward( + self, + audio: Tensor, + fake_audio: Tensor, + res_fake: List[Tuple[Tensor, Tensor]], + period_fake: List[Tuple[Tensor, Tensor]], + res_real: List[Tuple[Tensor, Tensor]], + period_real: List[Tuple[Tensor, Tensor]], + ) -> Tuple[ + Tensor, + Tensor, + Tensor, + Tensor, + Tensor, + Tensor, + ]: + r"""Calculate the losses for the generator and discriminator. + + Args: + audio (torch.Tensor): The real audio samples. + fake_audio (torch.Tensor): The generated audio samples. + res_fake (List[Tuple[Tensor, Tensor]]): The discriminator's output for the fake audio. + period_fake (List[Tuple[Tensor, Tensor]]): The discriminator's output for the fake audio in the period. + res_real (List[Tuple[Tensor, Tensor]]): The discriminator's output for the real audio. + period_real (List[Tuple[Tensor, Tensor]]): The discriminator's output for the real audio in the period. + + Returns: + tuple: A tuple containing the univnet loss, discriminator loss, STFT loss, score loss, ESR, SISDR, SNR and SDSDR losses. + """ + # Calculate the STFT loss + sc_loss, mag_loss = self.stft_criterion(fake_audio.squeeze(1), audio.squeeze(1)) + stft_loss = (sc_loss + mag_loss) * self.stft_lamb + + # Pad the fake audio to match the length of the real audio + padding = audio.shape[2] - fake_audio.shape[2] + fake_audio_padded = torch.nn.functional.pad(fake_audio, (0, padding)) + + esr_loss = self.esr_loss.forward(fake_audio_padded, audio) + snr_loss = self.snr_loss.forward(fake_audio_padded, audio) + + # Calculate the score loss + score_loss = torch.tensor(0.0, device=audio.device) + for _, score_fake in res_fake + period_fake: + score_loss += torch.mean(torch.pow(score_fake - 1.0, 2)) + + score_loss = score_loss / len(res_fake + period_fake) + + # Calculate the total generator loss + total_loss_gen = score_loss + stft_loss + esr_loss + snr_loss + + # Calculate the discriminator loss + total_loss_disc = torch.tensor(0.0, device=audio.device) + for (_, score_fake), (_, score_real) in zip( + res_fake + period_fake, res_real + period_real + ): + total_loss_disc += torch.mean(torch.pow(score_real - 1.0, 2)) + torch.mean( + torch.pow(score_fake, 2) + ) + + total_loss_disc = total_loss_disc / len(res_fake + period_fake) + + return ( + total_loss_gen, + total_loss_disc, + stft_loss, + score_loss, + esr_loss, + snr_loss, + ) diff --git a/training/loss/utils.py b/training/loss/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8e50a6bf9c2201689f047e2198ba463bea1339d1 --- /dev/null +++ b/training/loss/utils.py @@ -0,0 +1,19 @@ +import torch +from torch import Tensor + + +def sample_wise_min_max(x: Tensor) -> Tensor: + r"""Applies sample-wise min-max normalization to a tensor. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, num_samples, num_features). + + Returns: + torch.Tensor: Normalized tensor of the same shape as the input tensor. + """ + # Compute the maximum and minimum values of each sample in the batch + maximum = torch.amax(x, dim=(1, 2), keepdim=True) + minimum = torch.amin(x, dim=(1, 2), keepdim=True) + + # Apply sample-wise min-max normalization to the input tensor + return (x - minimum) / (maximum - minimum) diff --git a/training/np_tools.py b/training/np_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..09023d8277307563664961f8b735632515458a07 --- /dev/null +++ b/training/np_tools.py @@ -0,0 +1,91 @@ +from typing import List, Union + +import numpy as np + +# DEPRECATED: for dataset preprocessing use torch version. +# Here is datatype issue this code converts torch.float32 to numpy.float64 +# And it causes error in the model training: +# Error: RuntimeError: Input type (torch.cuda.DoubleTensor) and weight type (torch.cuda.FloatTensor) should be the same! + +def pad_1D(inputs: List[np.ndarray], pad_value: float = 0.0) -> np.ndarray: + r"""Pad a list of 1D numpy arrays to the same length. + + Args: + inputs (List[np.ndarray]): List of 1D numpy arrays to pad. + pad_value (float): Value to use for padding. Default is 0.0. + + Returns: + np.ndarray: Padded 2D numpy array of shape (len(inputs), max_len), where max_len is the length of the longest input array. + """ + + def pad_data(x: np.ndarray, length: int) -> np.ndarray: + r"""Pad a 1D numpy array with zeros to a specified length. + + Args: + x (np.ndarray): 1D numpy array to pad. + length (int): Length to pad the array to. + + Returns: + np.ndarray: Padded 1D numpy array of shape (length,). + """ + return np.pad( + x, (0, length - x.shape[0]), mode="constant", constant_values=pad_value, + ) + + max_len = max(len(x) for x in inputs) + return np.stack([pad_data(x, max_len) for x in inputs]) + + +def pad_2D( + inputs: List[np.ndarray], maxlen: Union[int, None] = None, pad_value: float = 0.0, +) -> np.ndarray: + r"""Pad a list of 2D numpy arrays to the same length. + + Args: + inputs (List[np.ndarray]): List of 2D numpy arrays to pad. + maxlen (Union[int, None]): Maximum length to pad the arrays to. If None, pad to the length of the longest array. Default is None. + pad_value (float): Value to use for padding. Default is 0.0. + + Returns: + np.ndarray: Padded 3D numpy array of shape (len(inputs), max_len, input_dim), where max_len is the maximum length of the input arrays, and input_dim is the dimension of the input arrays. + """ + + def pad(x: np.ndarray, max_len: int) -> np.ndarray: + r"""Pad a 2D numpy array with zeros to a specified length. + + Args: + x (np.ndarray): 2D numpy array to pad. + max_len (int): Maximum length to pad the array to. + + Returns: + np.ndarray: Padded 2D numpy array of shape (x.shape[0], max_len), where x.shape[0] is the number of rows in the input array. + """ + if np.shape(x)[1] > max_len: + raise ValueError("not max_len") + padding = np.ones((x.shape[0], max_len - np.shape(x)[1])) * pad_value + return np.concatenate((x, padding), 1) + + if maxlen: + output = np.stack([pad(x, maxlen) for x in inputs]) + else: + max_len = max(np.shape(x)[1] for x in inputs) + output = np.stack([pad(x, max_len) for x in inputs]) + return output + + +def pad_3D(inputs: Union[np.ndarray, List[np.ndarray]], B: int, T: int, L: int) -> np.ndarray: + r"""Pad a 3D numpy array to a specified shape. + + Args: + inputs (np.ndarray): 3D numpy array to pad. + B (int): Batch size to pad the array to. + T (int): Time steps to pad the array to. + L (int): Length to pad the array to. + + Returns: + np.ndarray: Padded 3D numpy array of shape (B, T, L), where B is the batch size, T is the time steps, and L is the length. + """ + inputs_padded = np.zeros((B, T, L), dtype=np.float32) + for i, input_ in enumerate(inputs): + inputs_padded[i, : np.shape(input_)[0], : np.shape(input_)[1]] = input_ + return inputs_padded diff --git a/training/preprocess/__init__.py b/training/preprocess/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b50f07be6e8bd8956068f0bf05e9c130bd5ecbee --- /dev/null +++ b/training/preprocess/__init__.py @@ -0,0 +1,7 @@ +from .audio import * +from .audio_processor import AudioProcessor +from .compute_yin import compute_yin +from .normalize_text import NormalizeText +from .preprocess_libritts import PreprocessLibriTTS +from .tacotron_stft import TacotronSTFT +from .wav2vec_aligner import Wav2VecAligner diff --git a/training/preprocess/audio.py b/training/preprocess/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..2095e49a3aca01bb84ef35de1b2db37da14810ab --- /dev/null +++ b/training/preprocess/audio.py @@ -0,0 +1,108 @@ +import sys +from typing import Tuple, Union + +import librosa +import numpy as np +import torch +import torchaudio + + +def stereo_to_mono(audio: torch.Tensor) -> torch.Tensor: + r"""Converts a stereo audio tensor to mono by taking the mean across channels. + + Args: + audio (torch.Tensor): Input audio tensor of shape (channels, samples). + + Returns: + torch.Tensor: Mono audio tensor of shape (1, samples). + """ + return torch.mean(audio, 0, True) + + +def resample(wav: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray: + r"""Resamples an audio waveform from the original sampling rate to the target sampling rate. + + Args: + wav (np.ndarray): The audio waveform to be resampled. + orig_sr (int): The original sampling rate of the audio waveform. + target_sr (int): The target sampling rate to resample the audio waveform to. + + Returns: + np.ndarray: The resampled audio waveform. + """ + return librosa.resample(wav, orig_sr=orig_sr, target_sr=target_sr) + + +def safe_load(path: str, sr: Union[int, None]) -> Tuple[np.ndarray, int]: + r"""Load an audio file from disk and return its content as a numpy array. + + Args: + path (str): The path to the audio file. + sr (int or None): The target sampling rate. If None, the original sampling rate is used. + + Returns: + Tuple[np.ndarray, int]: A tuple containing the audio content as a numpy array and the actual sampling rate. + """ + try: + audio, sr_actual = torchaudio.load(path) # type: ignore + if audio.shape[0] > 0: + audio = stereo_to_mono(audio) + audio = audio.squeeze(0) + if sr_actual != sr and sr is not None: + audio = resample(audio.numpy(), orig_sr=sr_actual, target_sr=sr) + sr_actual = sr + else: + audio = audio.numpy() + except Exception as e: + raise type(e)( + f"The following error happened loading the file {path} ... \n" + str(e), + ).with_traceback(sys.exc_info()[2]) + + return audio, sr_actual + + +def preprocess_audio( + audio: torch.Tensor, sr_actual: int, sr: Union[int, None], +) -> Tuple[torch.Tensor, int]: + r"""Preprocesses audio by converting stereo to mono, resampling if necessary, and returning the audio tensor and sample rate. + + Args: + audio (torch.Tensor): The audio tensor to preprocess. + sr_actual (int): The actual sample rate of the audio. + sr (Union[int, None]): The target sample rate to resample the audio to, if necessary. + + Returns: + Tuple[torch.Tensor, int]: The preprocessed audio tensor and sample rate. + """ + try: + if audio.shape[0] > 0: + audio = stereo_to_mono(audio) + audio = audio.squeeze(0) + if sr_actual != sr and sr is not None: + audio_np = resample(audio.numpy(), orig_sr=sr_actual, target_sr=sr) + # Convert back to torch tensor + audio = torch.from_numpy(audio_np) + sr_actual = sr + except Exception as e: + raise type(e)( + f"The following error happened while processing the audio ... \n {e!s}", + ).with_traceback(sys.exc_info()[2]) + + return audio, sr_actual + + +def normalize_loudness(wav: torch.Tensor) -> torch.Tensor: + r"""Normalize the loudness of an audio waveform. + + Args: + wav (torch.Tensor): The input waveform. + + Returns: + torch.Tensor: The normalized waveform. + + Examples: + >>> wav = np.array([1.0, 2.0, 3.0]) + >>> normalize_loudness(wav) + tensor([0.33333333, 0.66666667, 1. ]) + """ + return wav / torch.max(torch.abs(wav)) diff --git a/training/preprocess/audio_processor.py b/training/preprocess/audio_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..b8f92ec9e07db1dc2ad0a67ec7ae9af7064c8d92 --- /dev/null +++ b/training/preprocess/audio_processor.py @@ -0,0 +1,215 @@ +from librosa.filters import mel as librosa_mel_fn +import torch + + +class AudioProcessor: + r"""A class used to process audio signals and convert them into different representations. + + Attributes: + hann_window (dict): A dictionary to store the Hann window for different configurations. + mel_basis (dict): A dictionary to store the Mel basis for different configurations. + + Methods: + name_mel_basis(spec, n_fft, fmax): Generate a name for the Mel basis based on the FFT size, maximum frequency, data type, and device. + amp_to_db(magnitudes, C=1, clip_val=1e-5): Convert amplitude to decibels (dB). + db_to_amp(magnitudes, C=1): Convert decibels (dB) to amplitude. + wav_to_spec(y, n_fft, hop_length, win_length, center=False): Convert a waveform to a spectrogram and compute the magnitude. + wav_to_energy(y, n_fft, hop_length, win_length, center=False): Convert a waveform to a spectrogram and compute the energy. + spec_to_mel(spec, n_fft, num_mels, sample_rate, fmin, fmax): Convert a spectrogram to a Mel spectrogram. + wav_to_mel(y, n_fft, num_mels, sample_rate, hop_length, win_length, fmin, fmax, center=False): Convert a waveform to a Mel spectrogram. + """ + + def __init__(self): + self.hann_window = {} + self.mel_basis = {} + + @staticmethod + def name_mel_basis(spec: torch.Tensor, n_fft: int, fmax: int) -> str: + """Generate a name for the Mel basis based on the FFT size, maximum frequency, data type, and device. + + Args: + spec (torch.Tensor): The spectrogram tensor. + n_fft (int): The FFT size. + fmax (int): The maximum frequency. + + Returns: + str: The generated name for the Mel basis. + """ + n_fft_len = f"{n_fft}_{fmax}_{spec.dtype}_{spec.device}" + return n_fft_len + + @staticmethod + def amp_to_db(magnitudes: torch.Tensor, C: int = 1, clip_val: float = 1e-5) -> torch.Tensor: + r"""Convert amplitude to decibels (dB). + + Args: + magnitudes (Tensor): The amplitude magnitudes to convert. + C (int, optional): A constant value used in the conversion. Defaults to 1. + clip_val (float, optional): A value to clamp the magnitudes to avoid taking the log of zero. Defaults to 1e-5. + + Returns: + Tensor: The converted magnitudes in dB. + """ + return torch.log(torch.clamp(magnitudes, min=clip_val) * C) + + @staticmethod + def db_to_amp(magnitudes: torch.Tensor, C: int = 1) -> torch.Tensor: + r"""Convert decibels (dB) to amplitude. + + Args: + magnitudes (Tensor): The dB magnitudes to convert. + C (int, optional): A constant value used in the conversion. Defaults to 1. + + Returns: + Tensor: The converted magnitudes in amplitude. + """ + return torch.exp(magnitudes) / C + + def wav_to_spec( + self, + y: torch.Tensor, + n_fft: int, + hop_length: int, + win_length: int, + center: bool = False, + ) -> torch.Tensor: + r"""Convert a waveform to a spectrogram and compute the magnitude. + + Args: + y (Tensor): The input waveform. + n_fft (int): The FFT size. + hop_length (int): The hop (stride) size. + win_length (int): The window size. + center (bool, optional): Whether to pad `y` such that frames are centered. Defaults to False. + + Returns: + Tensor: The magnitude of the computed spectrogram. + """ + y = y.squeeze(1) + + dtype_device = str(y.dtype) + "_" + str(y.device) + wnsize_dtype_device = str(win_length) + "_" + dtype_device + if wnsize_dtype_device not in self.hann_window: + self.hann_window[wnsize_dtype_device] = torch.hann_window(win_length).to(dtype=y.dtype, device=y.device) + + y = torch.nn.functional.pad( + y.unsqueeze(1), + (int((n_fft - hop_length) / 2), int((n_fft - hop_length) / 2)), + mode="reflect", + ) + y = y.squeeze(1) + + spec = torch.stft( + y, + n_fft, + hop_length=hop_length, + win_length=win_length, + window=self.hann_window[wnsize_dtype_device], + center=center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + + spec = torch.view_as_real(spec) + + # Compute the magnitude + spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) + + return spec + + def wav_to_energy( + self, + y: torch.Tensor, + n_fft: int, + hop_length: int, + win_length: int, + center: bool = False, + ) -> torch.Tensor: + r"""Convert a waveform to a spectrogram and compute the energy. + + Args: + y (Tensor): The input waveform. + n_fft (int): The FFT size. + hop_length (int): The hop (stride) size. + win_length (int): The window size. + center (bool, optional): Whether to pad `y` such that frames are centered. Defaults to False. + + Returns: + Tensor: The energy of the computed spectrogram. + """ + spec = self.wav_to_spec(y, n_fft, hop_length, win_length, center=center) + spec = torch.norm(spec, dim=1, keepdim=True).squeeze(0) + + # Normalize the energy + return (spec - spec.mean()) / spec.std() + + def spec_to_mel( + self, + spec: torch.Tensor, + n_fft: int, + num_mels: int, + sample_rate: int, + fmin: int, + fmax: int, + ) -> torch.Tensor: + r"""Convert a spectrogram to a Mel spectrogram. + + Args: + spec (torch.Tensor): The input spectrogram of shape [B, C, T]. + n_fft (int): The FFT size. + num_mels (int): The number of Mel bands. + sample_rate (int): The sample rate of the audio. + fmin (int): The minimum frequency. + fmax (int): The maximum frequency. + + Returns: + torch.Tensor: The computed Mel spectrogram of shape [B, C, T]. + """ + mel_basis_key = self.name_mel_basis(spec, n_fft, fmax) + + if mel_basis_key not in self.mel_basis: + mel = librosa_mel_fn(sr=sample_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) + self.mel_basis[mel_basis_key] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) + + mel = torch.matmul(self.mel_basis[mel_basis_key], spec) + mel = self.amp_to_db(mel) + + return mel + + def wav_to_mel( + self, + y: torch.Tensor, + n_fft: int, + num_mels: int, + sample_rate: int, + hop_length: int, + win_length: int, + fmin: int, + fmax: int, + center: bool = False, + ) -> torch.Tensor: + r"""Convert a waveform to a Mel spectrogram. + + Args: + y (torch.Tensor): The input waveform. + n_fft (int): The FFT size. + num_mels (int): The number of Mel bands. + sample_rate (int): The sample rate of the audio. + hop_length (int): The hop (stride) size. + win_length (int): The window size. + fmin (int): The minimum frequency. + fmax (int): The maximum frequency. + center (bool, optional): Whether to pad `y` such that frames are centered. Defaults to False. + + Returns: + torch.Tensor: The computed Mel spectrogram. + """ + # Convert the waveform to a spectrogram + spec = self.wav_to_spec(y, n_fft, hop_length, win_length, center=center) + + # Convert the spectrogram to a Mel spectrogram + mel = self.spec_to_mel(spec, n_fft, num_mels, sample_rate, fmin, fmax) + + return mel diff --git a/training/preprocess/compute_yin.py b/training/preprocess/compute_yin.py new file mode 100644 index 0000000000000000000000000000000000000000..1e1db3519581445b13d7e55c14bfbb95ac33f4fc --- /dev/null +++ b/training/preprocess/compute_yin.py @@ -0,0 +1,243 @@ +from typing import List, Tuple + +import numpy as np +import torch +import torch.nn.functional as F + +####################################################################################### +# Original implementation from https://github.com/NVIDIA/mellotron/blob/master/yin.py # +####################################################################################### + + +def differenceFunction(x: np.ndarray, N: int, tau_max: int) -> np.ndarray: + r"""Compute the difference function of an audio signal. + + This function computes the difference function of an audio signal `x` using the algorithm described in equation (6) of [1]. The difference function is a measure of the similarity between the signal and a time-shifted version of itself, and is commonly used in pitch detection algorithms. + + This implementation uses the NumPy FFT functions to compute the difference function efficiently. + + Parameters + x (np.ndarray): The audio signal to compute the difference function for. + N (int): The length of the audio signal. + tau_max (int): The maximum integration window size to use. + + Returns + np.ndarray: The difference function of the audio signal. + + References + [1] A. de Cheveigné and H. Kawahara, "YIN, a fundamental frequency estimator for speech and music," The Journal of the Acoustical Society of America, vol. 111, no. 4, pp. 1917-1930, 2002. + """ + x = np.array(x, np.float64) + w = x.size + tau_max = min(tau_max, w) + x_cumsum = np.concatenate((np.array([0.0]), (x * x).cumsum())) + size = w + tau_max + p2 = (size // 32).bit_length() + nice_numbers = (16, 18, 20, 24, 25, 27, 30, 32) + size_pad = min(x * 2**p2 for x in nice_numbers if x * 2**p2 >= size) + fc = np.fft.rfft(x, size_pad) + conv = np.fft.irfft(fc * fc.conjugate())[:tau_max] + return x_cumsum[w : w - tau_max : -1] + x_cumsum[w] - x_cumsum[:tau_max] - 2 * conv + + +def cumulativeMeanNormalizedDifferenceFunction(df: np.ndarray, N: int) -> np.ndarray: + r"""Compute the cumulative mean normalized difference function (CMND) of a difference function. + + The CMND is defined as the element-wise product of the difference function with a range of values from 1 to N-1, + divided by the cumulative sum of the difference function up to that point, plus a small epsilon value to avoid + division by zero. The first element of the CMND is set to 1. + + Args: + df (np.ndarray): The difference function. + N (int): The length of the data. + + Returns: + np.ndarray: The cumulative mean normalized difference function. + + References: + [1] K. K. Paliwal and R. P. Sharma, "A robust algorithm for pitch detection in noisy speech signals," + Speech Communication, vol. 12, no. 3, pp. 249-263, 1993. + """ + cmndf = ( + df[1:] * range(1, N) / (np.cumsum(df[1:]).astype(float) + 1e-8) + ) # scipy method + return np.insert(cmndf, 0, 1) + + +def getPitch(cmdf: np.ndarray, tau_min: int, tau_max: int, harmo_th: float=0.1) -> int: + r"""Compute the fundamental period of a frame based on the Cumulative Mean Normalized Difference function (CMND). + + The CMND is a measure of the periodicity of a signal, and is computed as the cumulative mean normalized difference + function of the difference function of the signal. The fundamental period is the first value of the index `tau` + between `tau_min` and `tau_max` where the CMND is below the `harmo_th` threshold. If there are no such values, the + function returns 0 to indicate that the signal is unvoiced. + + Args: + cmdf (np.ndarray): The Cumulative Mean Normalized Difference function of the signal. + tau_min (int): The minimum period for speech. + tau_max (int): The maximum period for speech. + harmo_th (float, optional): The harmonicity threshold to determine if it is necessary to compute pitch + frequency. Defaults to 0.1. + + Returns: + int: The fundamental period of the signal, or 0 if the signal is unvoiced. + + References: + [1] K. K. Paliwal and R. P. Sharma, "A robust algorithm for pitch detection in noisy speech signals," + Speech Communication, vol. 12, no. 3, pp. 249-263, 1993. + """ + tau = tau_min + while tau < tau_max: + if cmdf[tau] < harmo_th: + while tau + 1 < tau_max and cmdf[tau + 1] < cmdf[tau]: + tau += 1 + return tau + tau += 1 + + return 0 # if unvoiced + + +def compute_yin( + sig_torch: torch.Tensor, + sr: int, + w_len: int = 512, + w_step: int = 256, + f0_min: int = 100, + f0_max: int = 500, + harmo_thresh: float = 0.1, +) -> Tuple[np.ndarray, List[float], List[float], List[float]]: + r"""Compute the Yin Algorithm for pitch detection on an audio signal. + + The Yin Algorithm is a widely used method for pitch detection in speech and music signals. It works by computing the + Cumulative Mean Normalized Difference function (CMND) of the difference function of the signal, and finding the first + minimum of the CMND below a given threshold. The fundamental period of the signal is then estimated as the inverse of + the lag corresponding to this minimum. + + Args: + sig_torch (torch.Tensor): The audio signal as a 1D numpy array of floats. + sr (int): The sampling rate of the signal. + w_len (int, optional): The size of the analysis window in samples. Defaults to 512. + w_step (int, optional): The size of the lag between two consecutive windows in samples. Defaults to 256. + f0_min (int, optional): The minimum fundamental frequency that can be detected in Hz. Defaults to 100. + f0_max (int, optional): The maximum fundamental frequency that can be detected in Hz. Defaults to 500. + harmo_thresh (float, optional): The threshold of detection. The algorithm returns the first minimum of the CMND + function below this threshold. Defaults to 0.1. + + Returns: + Tuple[np.ndarray, List[float], List[float], List[float]]: A tuple containing the following elements: + * pitches (np.ndarray): A 1D numpy array of fundamental frequencies estimated for each analysis window. + * harmonic_rates (List[float]): A list of harmonic rate values for each fundamental frequency value, which + can be interpreted as a confidence value. + * argmins (List[float]): A list of the minimums of the Cumulative Mean Normalized Difference Function for + each analysis window. + * times (List[float]): A list of the time of each estimation, in seconds. + + References: + [1] A. K. Jain, Fundamentals of Digital Image Processing, Prentice Hall, 1989. + [2] A. de Cheveigné and H. Kawahara, "YIN, a fundamental frequency estimator for speech and music," The Journal + of the Acoustical Society of America, vol. 111, no. 4, pp. 1917-1930, 2002. + """ + sig_torch = sig_torch.view(1, 1, -1) + sig_torch = F.pad( + sig_torch.unsqueeze(1), + (int((w_len - w_step) / 2), int((w_len - w_step) / 2), 0, 0), + mode="reflect", + ) + sig_torch_n: np.ndarray = sig_torch.view(-1).numpy() + + tau_min = int(sr / f0_max) + tau_max = int(sr / f0_min) + + timeScale = range( + 0, len(sig_torch_n) - w_len, w_step, + ) # time values for each analysis window + times = [t / float(sr) for t in timeScale] + frames = [sig_torch_n[t : t + w_len] for t in timeScale] + + pitches = [0.0] * len(timeScale) + harmonic_rates = [0.0] * len(timeScale) + argmins = [0.0] * len(timeScale) + + for i, frame in enumerate(frames): + # Compute YIN + df = differenceFunction(frame, w_len, tau_max) + cmdf = cumulativeMeanNormalizedDifferenceFunction(df, tau_max) + p = getPitch(cmdf, tau_min, tau_max, harmo_thresh) + + # Get results + if np.argmin(cmdf) > tau_min: + argmins[i] = float(sr / np.argmin(cmdf)) + if p != 0: # A pitch was found + pitches[i] = float(sr / p) + harmonic_rates[i] = cmdf[p] + else: # No pitch, but we compute a value of the harmonic rate + harmonic_rates[i] = min(cmdf) + + return np.array(pitches), harmonic_rates, argmins, times + + +def norm_interp_f0(f0: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + r"""Normalize and interpolate the fundamental frequency (f0) values. + + Args: + f0 (np.ndarray): The input f0 values. + + Returns: + Tuple[np.ndarray, np.ndarray]: A tuple containing the normalized f0 values and a boolean array indicating which values were interpolated. + + Examples: + >>> f0 = np.array([0, 100, 0, 200, 0]) + >>> norm_interp_f0(f0) + ( + np.array([100, 100, 150, 200, 200]), + np.array([True, False, True, False, True]), + ) + """ + uv: np.ndarray = f0 == 0 + if sum(uv) == len(f0): + f0[uv] = 0 + elif sum(uv) > 0: + f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv]) + return f0, uv + + +def compute_pitch( + sig_torch: torch.Tensor, + sr: int, + w_len: int = 1024, + w_step: int = 256, + f0_min: int = 50, + f0_max: int = 1000, + harmo_thresh: float = 0.25, +): + r"""Compute the pitch of an audio signal using the Yin algorithm. + + The Yin algorithm is a widely used method for pitch detection in speech and music signals. This function uses the + Yin algorithm to compute the pitch of the input audio signal, and then normalizes and interpolates the pitch values. + Returns the normalized and interpolated pitch values. + + Args: + sig_torch (torch.Tensor): The audio signal as a 1D numpy array of floats. + sr (int): The sampling rate of the signal. + w_len (int, optional): The size of the analysis window in samples. + w_step (int, optional): The size of the lag between two consecutive windows in samples. + f0_min (int, optional): The minimum fundamental frequency that can be detected in Hz. + f0_max (int, optional): The maximum fundamental frequency that can be detected in Hz. + harmo_thresh (float, optional): The threshold of detection. The algorithm returns the first minimum of the CMND function below this threshold. + + Returns: + np.ndarray: The normalized and interpolated pitch values of the audio signal. + """ + pitch, _, _, _ = compute_yin( + sig_torch, + sr=sr, + w_len=w_len, + w_step=w_step, + f0_min=f0_min, + f0_max=f0_max, + harmo_thresh=harmo_thresh, + ) + + pitch, _ = norm_interp_f0(pitch) + + return pitch diff --git a/training/preprocess/normalize_text.py b/training/preprocess/normalize_text.py new file mode 100644 index 0000000000000000000000000000000000000000..9bde333334ba89ded4d620cb11e7fc72c9624ae8 --- /dev/null +++ b/training/preprocess/normalize_text.py @@ -0,0 +1,110 @@ +import re + +from nemo_text_processing.text_normalization.normalize import Normalizer +from unidecode import unidecode + + +class NormalizeText: + r"""NVIDIA NeMo is a conversational AI toolkit built for researchers working on automatic speech recognition (ASR), text-to-speech synthesis (TTS), large language models (LLMs), and natural language processing (NLP). The primary objective of NeMo is to help researchers from industry and academia to reuse prior work (code and pretrained models) and make it easier to create new conversational AI models. + + This class normalize the characters in the input text and normalize the input text with the `nemo_text_processing`. + + Args: + lang (str): The language code to use for normalization. Defaults to "en". + + Attributes: + lang (str): The language code to use for normalization. Defaults to "en". + model (Normalizer): The `nemo_text_processing` Normalizer model. + + Methods: + byte_encode(word: str) -> list: Encode a word as a list of bytes. + normalize_chars(text: str) -> str: Normalize the characters in the input text. + __call__(text: str) -> str: Normalize the input text with the `nemo_text_processing`. + + Examples: + >>> from training.preprocess.normilize_text import NormalizeText + >>> normilize_text = NormalizeText() + >>> normilize_text("It’s a beautiful day…") + "It's a beautiful day." + """ + + def __init__(self, lang: str = "en"): + r"""Initialize a new instance of the NormalizeText class. + + Args: + lang (str): The language code to use for normalization. Defaults to "en". + + """ + self.lang = lang + + self.model = Normalizer(input_case="cased", lang=lang) + + def byte_encode(self, word: str) -> list: + r"""Encode a word as a list of bytes. + + Args: + word (str): The word to encode. + + Returns: + list: A list of bytes representing the encoded word. + + """ + text = word.strip() + return list(text.encode("utf-8")) + + def normalize_chars(self, text: str) -> str: + r"""Normalize the characters in the input text. + + Args: + text (str): The input text to normalize. + + Returns: + str: The normalized text. + + Examples: + >>> normalize_chars("It’s a beautiful day…") + "It's a beautiful day." + + """ + # Define the character mapping + char_mapping = { + ord("’"): ord("'"), + ord("”"): ord("'"), + ord("…"): ord("."), + ord("„"): ord("'"), + ord("“"): ord("'"), + ord('"'): ord("'"), + ord("–"): ord("-"), + ord("—"): ord("-"), + ord("«"): ord("'"), + ord("»"): ord("'"), + } + + # Add unicode normalization as additional garanty and normalize the characters using translate() method + normalized_string = unidecode(text).translate(char_mapping) + + # Remove redundant multiple characters + # TODO: Maybe there is some effect on duplication? + return re.sub(r"(\.|\!|\?|\-)\1+", r"\1", normalized_string) + + def __call__(self, text: str) -> str: + r"""Normalize the input text with the `nemo_text_processing`. + + Args: + text (str): The input text to normalize. + + Returns: + str: The normalized text. + + """ + text = self.normalize_chars(text) + # return self.model.normalize(text) + + # Split the text into lines + lines = text.split("\n") + normalized_lines = self.model.normalize_list(lines) + + # TODO: check this! + # Join the normalized lines, replace \n with . and return the result + result = ". ".join(normalized_lines) + return result diff --git a/training/preprocess/preprocess_libritts.py b/training/preprocess/preprocess_libritts.py new file mode 100644 index 0000000000000000000000000000000000000000..253ef759003ea1d9afcf0eb414112e2bd3e7c307 --- /dev/null +++ b/training/preprocess/preprocess_libritts.py @@ -0,0 +1,307 @@ +from dataclasses import dataclass +import math +import random +from typing import Any, List, Tuple, Union + +import numpy as np +from scipy.stats import betabinom +import torch +import torch.nn.functional as F + +from models.config import PreprocessingConfig, VocoderBasicConfig, get_lang_map + +from .audio import normalize_loudness, preprocess_audio +from .audio_processor import AudioProcessor +from .compute_yin import compute_yin, norm_interp_f0 +from .normalize_text import NormalizeText +from .tacotron_stft import TacotronSTFT +from .tokenizer_ipa_espeak import TokenizerIpaEspeak as TokenizerIPA + + +@dataclass +class PreprocessForAcousticResult: + wav: torch.Tensor + mel: torch.Tensor + pitch: torch.Tensor + phones_ipa: Union[str, List[str]] + phones: torch.Tensor + attn_prior: torch.Tensor + energy: torch.Tensor + raw_text: str + normalized_text: str + speaker_id: int + chapter_id: str | int + utterance_id: str + pitch_is_normalized: bool + + +class PreprocessLibriTTS: + r"""Preprocessing PreprocessLibriTTS audio and text data for use with a TacotronSTFT model. + + Args: + preprocess_config (PreprocessingConfig): The preprocessing configuration. + lang (str): The language of the input text. + + Attributes: + min_seconds (float): The minimum duration of audio clips in seconds. + max_seconds (float): The maximum duration of audio clips in seconds. + hop_length (int): The hop length of the STFT. + sampling_rate (int): The sampling rate of the audio. + use_audio_normalization (bool): Whether to normalize the loudness of the audio. + tacotronSTFT (TacotronSTFT): The TacotronSTFT object used for computing mel spectrograms. + min_samples (int): The minimum number of audio samples in a clip. + max_samples (int): The maximum number of audio samples in a clip. + """ + + def __init__( + self, + preprocess_config: PreprocessingConfig, + lang: str = "en", + ): + super().__init__() + + lang_map = get_lang_map(lang) + + self.phonemizer_lang = lang_map.phonemizer + normilize_text_lang = lang_map.nemo + + self.normilize_text = NormalizeText(normilize_text_lang) + self.tokenizer = TokenizerIPA(lang) + self.vocoder_train_config = VocoderBasicConfig() + + self.preprocess_config = preprocess_config + + self.sampling_rate = self.preprocess_config.sampling_rate + self.use_audio_normalization = self.preprocess_config.use_audio_normalization + + self.hop_length = self.preprocess_config.stft.hop_length + self.filter_length = self.preprocess_config.stft.filter_length + self.mel_fmin = self.preprocess_config.stft.mel_fmin + self.win_length = self.preprocess_config.stft.win_length + + self.tacotronSTFT = TacotronSTFT( + filter_length=self.filter_length, + hop_length=self.hop_length, + win_length=self.preprocess_config.stft.win_length, + n_mel_channels=self.preprocess_config.stft.n_mel_channels, + sampling_rate=self.sampling_rate, + mel_fmin=self.mel_fmin, + mel_fmax=self.preprocess_config.stft.mel_fmax, + center=False, + ) + + min_seconds, max_seconds = ( + self.preprocess_config.min_seconds, + self.preprocess_config.max_seconds, + ) + + self.min_samples = int(self.sampling_rate * min_seconds) + self.max_samples = int(self.sampling_rate * max_seconds) + + self.audio_processor = AudioProcessor() + + def beta_binomial_prior_distribution( + self, + phoneme_count: int, + mel_count: int, + scaling_factor: float = 1.0, + ) -> torch.Tensor: + r"""Computes the beta-binomial prior distribution for the attention mechanism. + + Args: + phoneme_count (int): Number of phonemes in the input text. + mel_count (int): Number of mel frames in the input mel-spectrogram. + scaling_factor (float, optional): Scaling factor for the beta distribution. Defaults to 1.0. + + Returns: + torch.Tensor: A 2D tensor containing the prior distribution. + """ + P, M = phoneme_count, mel_count + x = np.arange(0, P) + mel_text_probs = [] + for i in range(1, M + 1): + a, b = scaling_factor * i, scaling_factor * (M + 1 - i) + rv: Any = betabinom(P, a, b) + mel_i_prob = rv.pmf(x) + mel_text_probs.append(mel_i_prob) + return torch.from_numpy(np.array(mel_text_probs)) + + def acoustic( + self, + row: Tuple[torch.Tensor, int, str, str, int, str | int, str], + ) -> Union[None, PreprocessForAcousticResult]: + r"""Preprocesses audio and text data for use with a TacotronSTFT model. + + Args: + row (Tuple[torch.FloatTensor, int, str, str, int, str | int, str]): The input row. The row is a tuple containing the following elements: (audio, sr_actual, raw_text, normalized_text, speaker_id, chapter_id, utterance_id). + + Returns: + dict: A dictionary containing the preprocessed audio and text data. + + Examples: + >>> preprocess_audio = PreprocessAudio("english_only") + >>> audio = torch.randn(1, 44100) + >>> sr_actual = 44100 + >>> raw_text = "Hello, world!" + >>> output = preprocess_audio(audio, sr_actual, raw_text) + >>> output.keys() + dict_keys(['wav', 'mel', 'pitch', 'phones', 'raw_text', 'normalized_text', 'speaker_id', 'chapter_id', 'utterance_id', 'pitch_is_normalized']) + """ + ( + audio, + sr_actual, + raw_text, + normalized_text, + speaker_id, + chapter_id, + utterance_id, + ) = row + + wav, sampling_rate = preprocess_audio(audio, sr_actual, self.sampling_rate) + + # TODO: check this, maybe you need to move it to some other place + # TODO: maybe we can increate the max_samples ? + # if wav.shape[0] < self.min_samples or wav.shape[0] > self.max_samples: + # return None + + if self.use_audio_normalization: + wav = normalize_loudness(wav) + + normalized_text = self.normilize_text(normalized_text) + + # NOTE: fixed version of tokenizer with punctuation + phones_ipa, phones = self.tokenizer(normalized_text) + + # Convert to tensor + phones = torch.Tensor(phones) + + mel_spectrogram = self.tacotronSTFT.get_mel_from_wav(wav) + + # Skipping small sample due to the mel-spectrogram containing less than self.mel_fmin frames + # if mel_spectrogram.shape[1] < self.mel_fmin: + # return None + + # Text is longer than mel, will be skipped due to monotonic alignment search + if phones.shape[0] >= mel_spectrogram.shape[1]: + return None + + pitch, _, _, _ = compute_yin( + wav, + sr=sampling_rate, + w_len=self.filter_length, + w_step=self.hop_length, + f0_min=50, + f0_max=1000, + harmo_thresh=0.25, + ) + + pitch, _ = norm_interp_f0(pitch) + + if np.sum(pitch != 0) <= 1: + return None + + pitch = torch.from_numpy(pitch) + + # TODO this shouldnt be necessary, currently pitch sometimes has 1 less frame than spectrogram, + # We should find out why + mel_spectrogram = mel_spectrogram[:, : pitch.shape[0]] + + attn_prior = self.beta_binomial_prior_distribution( + phones.shape[0], + mel_spectrogram.shape[1], + ).T + + assert pitch.shape[0] == mel_spectrogram.shape[1], ( + pitch.shape, + mel_spectrogram.shape[1], + ) + + energy = self.audio_processor.wav_to_energy( + wav.unsqueeze(0), + self.filter_length, + self.hop_length, + self.win_length, + ) + + return PreprocessForAcousticResult( + wav=wav, + mel=mel_spectrogram, + pitch=pitch, + attn_prior=attn_prior, + energy=energy, + phones_ipa=phones_ipa, + phones=phones, + raw_text=raw_text, + normalized_text=normalized_text, + speaker_id=speaker_id, + chapter_id=chapter_id, + utterance_id=utterance_id, + # TODO: check the pitch normalization process + pitch_is_normalized=False, + ) + + def univnet(self, row: Tuple[torch.Tensor, int, str, str, int, str | int, str]): + r"""Preprocesses audio data for use with a UnivNet model. + + This method takes a row of data, extracts the audio and preprocesses it. + It then selects a random segment from the preprocessed audio and its corresponding mel spectrogram. + + Args: + row (Tuple[torch.FloatTensor, int, str, str, int, str | int, str]): The input row. The row is a tuple containing the following elements: (audio, sr_actual, raw_text, normalized_text, speaker_id, chapter_id, utterance_id). + + Returns: + Tuple[torch.Tensor, torch.Tensor, int]: A tuple containing the selected segment of the mel spectrogram, the corresponding audio segment, and the speaker ID. + + Examples: + >>> preprocess = PreprocessLibriTTS() + >>> audio = torch.randn(1, 44100) + >>> sr_actual = 44100 + >>> speaker_id = 0 + >>> mel, audio_segment, speaker_id = preprocess.preprocess_univnet((audio, sr_actual, "", "", speaker_id, 0, "")) + """ + ( + audio, + sr_actual, + _, + _, + speaker_id, + _, + _, + ) = row + + segment_size = self.vocoder_train_config.segment_size + frames_per_seg = math.ceil(segment_size / self.hop_length) + + wav, _ = preprocess_audio(audio, sr_actual, self.sampling_rate) + + if self.use_audio_normalization: + wav = normalize_loudness(wav) + + mel_spectrogram = self.tacotronSTFT.get_mel_from_wav(wav) + + if wav.shape[0] < segment_size: + wav = F.pad( + wav, + (0, segment_size - wav.shape[0]), + "constant", + ) + + if mel_spectrogram.shape[1] < frames_per_seg: + mel_spectrogram = F.pad( + mel_spectrogram, + (0, frames_per_seg - mel_spectrogram.shape[1]), + "constant", + ) + + from_frame = random.randint(0, mel_spectrogram.shape[1] - frames_per_seg) + + # Skip last frame, otherwise errors are thrown, find out why + if from_frame > 0: + from_frame -= 1 + + till_frame = from_frame + frames_per_seg + + mel_spectrogram = mel_spectrogram[:, from_frame:till_frame] + wav = wav[from_frame * self.hop_length : till_frame * self.hop_length] + + return mel_spectrogram, wav, speaker_id diff --git a/training/preprocess/tacotron_stft.py b/training/preprocess/tacotron_stft.py new file mode 100644 index 0000000000000000000000000000000000000000..4e23c723552dff2a4678580907b3907bfefc2fd8 --- /dev/null +++ b/training/preprocess/tacotron_stft.py @@ -0,0 +1,160 @@ +from typing import Optional, Tuple + +import librosa +import torch +from torch.nn import Module + + +class TacotronSTFT(Module): + def __init__( + self, + filter_length: int, + hop_length: int, + win_length: int, + n_mel_channels: int, + sampling_rate: int, + center: bool, + mel_fmax: Optional[int], + mel_fmin: float = 0.0, + ): + r"""TacotronSTFT module that computes mel-spectrograms from a batch of waves. + + Args: + filter_length (int): Length of the filter window. + hop_length (int): Number of samples between successive frames. + win_length (int): Size of the STFT window. + n_mel_channels (int): Number of mel bins. + sampling_rate (int): Sampling rate of the input waveforms. + mel_fmin (int or None): Minimum frequency for the mel filter bank. + mel_fmax (int or None): Maximum frequency for the mel filter bank. + center (bool): Whether to pad the input signal on both sides. + """ + super().__init__() + + self.n_mel_channels = n_mel_channels + self.sampling_rate = sampling_rate + self.n_fft = filter_length + self.hop_size = hop_length + self.win_size = win_length + self.fmin = mel_fmin + self.fmax = mel_fmax + self.center = center + + # Define the mel filterbank + mel = librosa.filters.mel( + sr=sampling_rate, + n_fft=filter_length, + n_mels=n_mel_channels, + fmin=mel_fmin, + fmax=mel_fmax, + ) + + mel_basis = torch.from_numpy(mel).float() + + # Define the Hann window + hann_window = torch.hann_window(win_length) + + self.register_buffer("mel_basis", mel_basis) + self.register_buffer("hann_window", hann_window) + + def _spectrogram(self, y: torch.Tensor) -> torch.Tensor: + r"""Computes the linear spectrogram of a batch of waves. + + Args: + y (torch.Tensor): Input waveforms. + + Returns: + torch.Tensor: Linear spectrogram. + """ + assert torch.min(y.data) >= -1 + assert torch.max(y.data) <= 1 + + y = torch.nn.functional.pad( + y.unsqueeze(1), + ( + int((self.n_fft - self.hop_size) / 2), + int((self.n_fft - self.hop_size) / 2), + ), + mode="reflect", + ) + y = y.squeeze(1) + spec = torch.stft( + y, + self.n_fft, + hop_length=self.hop_size, + win_length=self.win_size, + window=self.hann_window, # type: ignore + center=self.center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + return torch.view_as_real(spec) + + def linear_spectrogram(self, y: torch.Tensor) -> torch.Tensor: + r"""Computes the linear spectrogram of a batch of waves. + + Args: + y (torch.Tensor): Input waveforms. + + Returns: + torch.Tensor: Linear spectrogram. + """ + spec = self._spectrogram(y) + return torch.norm(spec, p=2, dim=-1) + + def forward(self, y: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Computes mel-spectrograms from a batch of waves. + + Args: + y (torch.FloatTensor): Input waveforms with shape (B, T) in range [-1, 1] + + Returns: + torch.FloatTensor: Spectrogram of shape (B, n_spech_channels, T) + torch.FloatTensor: Mel-spectrogram of shape (B, n_mel_channels, T) + """ + spec = self._spectrogram(y) + + spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) + + mel = torch.matmul(self.mel_basis, spec) # type: ignore + mel = self.spectral_normalize_torch(mel) + + return spec, mel + + def spectral_normalize_torch(self, magnitudes: torch.Tensor) -> torch.Tensor: + r"""Applies dynamic range compression to magnitudes. + + Args: + magnitudes (torch.Tensor): Input magnitudes. + + Returns: + torch.Tensor: Output magnitudes. + """ + return self.dynamic_range_compression_torch(magnitudes) + + def dynamic_range_compression_torch( + self, + x: torch.Tensor, + C: int = 1, + clip_val: float = 1e-5, + ) -> torch.Tensor: + r"""Applies dynamic range compression to x. + + Args: + x (torch.Tensor): Input tensor. + C (float): Compression factor. + clip_val (float): Clipping value. + + Returns: + torch.Tensor: Output tensor. + """ + return torch.log(torch.clamp(x, min=clip_val) * C) + + # NOTE: audio np.ndarray changed to torch.FloatTensor! + def get_mel_from_wav(self, audio: torch.Tensor) -> torch.Tensor: + audio_tensor = audio.unsqueeze(0) + with torch.no_grad(): + _, melspec = self.forward(audio_tensor) + return melspec.squeeze(0) diff --git a/training/preprocess/tests/__init__.py b/training/preprocess/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/training/preprocess/tests/test_audio.py b/training/preprocess/tests/test_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..fbef203a4c907a296a4a77630ffb8ce7a9ec82bb --- /dev/null +++ b/training/preprocess/tests/test_audio.py @@ -0,0 +1,124 @@ +import math +import unittest + +import numpy as np +import torch + +from training.preprocess.audio import ( + normalize_loudness, + preprocess_audio, + resample, + stereo_to_mono, +) + + +# Create a class to test the ComputePitch class +class TestAudio(unittest.TestCase): + def setUp(self): + np.random.seed(0) + + def test_stereo_to_mono(self): + # Test the stereo_to_mono function with a simple example + audio = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + expected_output = torch.tensor([[2.5, 3.5, 4.5]]) + actual_output = stereo_to_mono(audio) + + self.assertTrue(torch.allclose(actual_output, expected_output)) + + # Test the stereo_to_mono function with a larger example + audio = torch.randn(2, 44100) + expected_output = torch.mean(audio, 0, True) + actual_output = stereo_to_mono(audio) + self.assertTrue(torch.allclose(actual_output, expected_output)) + + # Test the stereo_to_mono function with a zero-dimensional tensor + audio = torch.tensor(1.0) + expected_output = torch.tensor([[1.0]]) + actual_output = stereo_to_mono(audio) + self.assertTrue(torch.allclose(actual_output, expected_output)) + + # Test the stereo_to_mono function with a one-dimensional tensor + audio = torch.tensor([1.0, 2.0, 3.0]) + expected_output = torch.tensor([2.0]) + actual_output = stereo_to_mono(audio) + self.assertTrue(torch.allclose(actual_output, expected_output)) + + # Test the stereo_to_mono function with a three-dimensional tensor + audio = torch.randn(2, 3, 44100) + expected_output = torch.mean(audio, 0, True) + actual_output = stereo_to_mono(audio) + self.assertTrue(torch.allclose(actual_output, expected_output)) + + def test_resample(self): + # Test the resample function with a simple example + wav = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + orig_sr = 44100 + target_sr = 22050 + + # Test the output shape + expected_shape = math.ceil( + len(wav) / 2, + ) + actual_output = resample(wav, orig_sr, target_sr) + self.assertEqual(actual_output.shape, (expected_shape,)) + + def test_preprocess_audio(self): + # Test the preprocess_audio function with a simple example + audio = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + sr_actual = 44100 + sr = 22050 + + # Test the output shape + expected_shape = torch.Size([2]) + actual_output, actual_sr = preprocess_audio(audio, sr_actual, sr) + self.assertEqual(actual_output.shape, expected_shape) + + # Test the output values + self.assertEqual(actual_sr, sr) + + # Test the preprocess_audio function with a three-dimensional tensor + audio = torch.randn(2, 3, 44100) + sr_actual = 44100 + sr = 22050 + + # Test the output shape + expected_shape = torch.Size([3, len(audio[0][0]) // 2]) + actual_output, actual_sr = preprocess_audio(audio, sr_actual, sr) + + self.assertEqual(actual_output.shape, expected_shape) + self.assertEqual(actual_sr, sr) + + def test_normalize_loudness(self): + # Test the normalize_loudness function with a simple example + wav = torch.tensor([1.0, 2.0, 3.0]) + expected_output = torch.tensor([0.33333333, 0.66666667, 1.0]) + actual_output = normalize_loudness(wav) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + # Test the normalize_loudness function with a larger example + wav = torch.randn(44100) + expected_output = wav / torch.max(torch.abs(wav)) + actual_output = normalize_loudness(wav) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + # Test the normalize_loudness function with a zero-dimensional array + wav = torch.tensor(1.0) + expected_output = torch.tensor(1.0) + actual_output = normalize_loudness(wav) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + # Test the normalize_loudness function with a one-dimensional array + wav = torch.tensor([1.0, 2.0, 3.0]) + expected_output = torch.tensor([0.33333333, 0.66666667, 1.0]) + actual_output = normalize_loudness(wav) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + # Test the normalize_loudness function with a three-dimensional array + wav = torch.randn(2, 3, 44100) + expected_output = wav / torch.max(torch.abs(wav)) + actual_output = normalize_loudness(wav) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_audio_preprocessor.py b/training/preprocess/tests/test_audio_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..41dc11f1035cdc9ef3bd8d964dc1cfd9b2f6eb41 --- /dev/null +++ b/training/preprocess/tests/test_audio_preprocessor.py @@ -0,0 +1,52 @@ +import unittest + +import torch + +from training.preprocess.audio_processor import AudioProcessor + + +class TestAudioProcessor(unittest.TestCase): + def setUp(self): + self.audio_processor = AudioProcessor() + self.y = torch.randn(1, 22050) # 1 second of audio at 22050Hz + self.n_fft = 2048 + self.hop_length = 512 + self.win_length = 2048 + self.num_mels = 128 + self.sample_rate = 22050 + self.fmin = 0 + self.fmax = 8000 + + def test_name_mel_basis(self): + spec = torch.randn(1, 1025, 100) + name = self.audio_processor.name_mel_basis(spec, self.n_fft, self.fmax) + self.assertEqual(name, f"{self.n_fft}_{self.fmax}_{spec.dtype}_{spec.device}") + + def test_amp_to_db_and_db_to_amp(self): + magnitudes = torch.abs(torch.randn(1, 100)) + db = self.audio_processor.amp_to_db(magnitudes) + amp = self.audio_processor.db_to_amp(db) + self.assertTrue(torch.allclose(magnitudes, amp, atol=1e-4)) + + def test_wav_to_spec(self): + spec = self.audio_processor.wav_to_spec(self.y, self.n_fft, self.hop_length, self.win_length) + self.assertEqual(spec.shape, (1, self.n_fft // 2 + 1, self.num_mels // 3 + 1)) + + def test_wav_to_energy(self): + energy = self.audio_processor.wav_to_energy(self.y, self.n_fft, self.hop_length, self.win_length) + self.assertEqual(energy.shape, (1, self.num_mels // 3 + 1)) + + def test_spec_to_mel(self): + spec = torch.randn(1, 1025, 100) + mel = self.audio_processor.spec_to_mel( + spec, self.n_fft, self.num_mels, self.sample_rate, self.fmin, self.fmax, + ) + self.assertEqual(mel.shape, (1, self.num_mels, 100)) + + def test_wav_to_mel(self): + mel = self.audio_processor.wav_to_mel(self.y, self.n_fft, self.num_mels, self.sample_rate, self.hop_length, self.win_length, self.fmin, self.fmax) + self.assertEqual(mel.shape, (1, self.num_mels, self.num_mels // 3 + 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_compute_yin.py b/training/preprocess/tests/test_compute_yin.py new file mode 100644 index 0000000000000000000000000000000000000000..7098f6be2e71bc855edaa05c27adc254f6353716 --- /dev/null +++ b/training/preprocess/tests/test_compute_yin.py @@ -0,0 +1,114 @@ +import unittest + +import numpy as np +import torch + +from training.preprocess.compute_yin import ( + compute_yin, + cumulativeMeanNormalizedDifferenceFunction, + differenceFunction, + getPitch, + norm_interp_f0, +) + + +# Create a class to test the ComputePitch class +class TestComputeYin(unittest.TestCase): + def setUp(self): + np.random.seed(0) + + def test_difference_function(self): + # Test the differenceFunction function with a simple example + x = np.array([0.0, 1.0, 2.0, 3.0, 4.0]) + N = len(x) + tau_max = 3 + + # Test the output shape + expected_shape = (tau_max,) + actual_shape = differenceFunction(x, N, tau_max).shape + + self.assertTrue(actual_shape == expected_shape) + + # Test the output values + expected_output = np.array([0.0, 4.0, 12.0]) + actual_output = differenceFunction(x, N, tau_max) + + np.testing.assert_equal(actual_output, expected_output) + + # Test the function with a larger example + x = np.random.randn(1000) + N = len(x) + tau_max = 100 + + # Test the output shape + expected_shape = (tau_max,) + actual_shape = differenceFunction(x, N, tau_max).shape + + self.assertTrue(actual_shape == expected_shape) + + # Test the output values + expected_output = np.zeros(tau_max) + for tau in range(1, tau_max): + expected_output[tau] = np.sum((x[: N - tau] - x[tau:N]) ** 2) + + actual_output = differenceFunction(x, N, tau_max) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + def test_get_pitch(self): + # Test the getPitch function with a simple example + cmdf = np.array([1.0, 0.5, 0.2, 0.1, 0.05]) + tau_min = 1 + tau_max = 5 + harmo_th = 0.3 + + # Test the output value when there is a value below the threshold + expected_output = 4 + actual_output = getPitch(cmdf, tau_min, tau_max, harmo_th) + self.assertEqual(actual_output, expected_output) + + # Test the output value when there are no values below the threshold + cmdf = np.array([1.0, 0.9, 0.8, 0.7, 0.6]) + expected_output = 0 + actual_output = getPitch(cmdf, tau_min, tau_max, harmo_th) + self.assertEqual(actual_output, expected_output) + + # Test the output value when there are no values below the threshold + cmdf = np.random.rand(100) + cmdf[tau_min:tau_max] = 0.5 + expected_output = 0 + actual_output = getPitch(cmdf, tau_min, tau_max, harmo_th) + self.assertEqual(actual_output, expected_output) + + def test_cumulative_mean_normalized_difference_function(self): + # Test the function with a simple example + df = np.array([1, 2, 3, 4, 5]) + N = len(df) + + # Test the output values + expected_output = np.array([1.0, 1.0, 1.2, 1.333333, 1.428571]) + actual_output = cumulativeMeanNormalizedDifferenceFunction(df, N) + np.testing.assert_allclose(actual_output, expected_output, rtol=1e-6, atol=1e-6) + + def test_compute_yin(self): + # Test the function with a simple example + sig = torch.from_numpy(np.sin(2 * np.pi * 440 * np.arange(44100) / 44100)) + sr = 44100 + + actual_output = compute_yin(sig, sr) + + # Check the result + expected_output = np.load("mocks/test_compute_yin.npy") + + np.testing.assert_allclose(actual_output, expected_output) + + def test_norm_interp_f0(self): + # Test the norm_interp_f0 function with a simple example + f0 = np.array([0, 100, 0, 200, 0]) + actual_output = norm_interp_f0(f0) + expected_output = ( + np.array([100, 100, 150, 200, 200]), + np.array([True, False, True, False, True]), + ) + np.testing.assert_allclose(actual_output[0], expected_output[0]) + # Test the norm_interp_f0 function with a zero-dimensional array + np.testing.assert_array_equal(actual_output[1], expected_output[1]) diff --git a/training/preprocess/tests/test_normalize_text.py b/training/preprocess/tests/test_normalize_text.py new file mode 100644 index 0000000000000000000000000000000000000000..034eba4499b350328c3c9153a3de72a3e02f8e2b --- /dev/null +++ b/training/preprocess/tests/test_normalize_text.py @@ -0,0 +1,137 @@ +import unittest + +import numpy as np + +from training.preprocess.normalize_text import NormalizeText + + +# Create a class to test the ComputePitch class +class TestTextPreprocess(unittest.TestCase): + def setUp(self): + np.random.seed(0) + self.normalizer = NormalizeText() + + def test_byte_encode(self): + # Test with a simple word + word = "hello" + expected_output = [104, 101, 108, 108, 111] + self.assertTrue(self.normalizer.byte_encode(word) == expected_output) + + # Test with a word containing non-ASCII characters + word = "héllo" + expected_output = [104, 195, 169, 108, 108, 111] + self.assertTrue(self.normalizer.byte_encode(word) == expected_output) + + # Test with an empty string + word = "" + expected_output = [] + self.assertTrue(self.normalizer.byte_encode(word) == expected_output) + + def test_normalize_chars(self): + # Test case 1: Test basic character normalization + input_text = "It’s a beautiful day…" + expected_output = "It's a beautiful day." + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 2: Test character normalization with multiple dots + input_text = "Hello..... world!!!!" + expected_output = "Hello. world!" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 3: Test character normalization with multiple exclamation marks + input_text = "Wow!!!!! This is amazing?????" + expected_output = "Wow! This is amazing?" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 4: Test character normalization with multiple question marks + input_text = "What????? I don't understand!????" + expected_output = "What? I don't understand!?" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 5: Test character normalization with multiple quotes + input_text = "He said, “I don’t know…”" + expected_output = "He said, 'I don't know.'" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 6: Test character normalization with multiple dashes + input_text = "This is a long--sentence" + expected_output = "This is a long-sentence" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + # Test case 7: Test character normalization with mixed characters + input_text = "It’s a beautiful day… What????? I don't understand!!!!!" + expected_output = "It's a beautiful day. What? I don't understand!" + self.assertEqual(self.normalizer.normalize_chars(input_text), expected_output) + + def test_normalize(self): + # Test case 1: Test basic text normalization + input_text = r"""It’s a beautiful day… Hello..... World!!!! Wow!!!!! This is amazing????? He said, “I don’t know…”. It’s a beautiful day… What????? I don't understand!!!!!""" + + expected_output = r"""It's a beautiful day. Hello. World! Wow! This is amazing? He said, 'I don't know.'. It's a beautiful day. What? I don't understand!""" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 2: Test text normalization with multiple dots + input_text = "Hello..... World!!!!" + expected_output = "Hello. World!" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 3: numbers + input_text = "1234567890" + expected_output = "one two three four five six seven eight nine zero" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 4: Complicated case + input_text = ( + "Mr. Smith paid $111 in U.S.A. on Dec. 17th. We paid $123 for this desk." + ) + expected_output = r"""mister Smith paid one hundred and eleven dollars in USA on december seventeenth. We paid one hundred and twenty three dollars for this desk.""" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 5: Complicated case 2 + input_text = "St. Patrick’s Day, spend $123 for this desk." + expected_output = r"""Saint Patrick's Day, spend one hundred and twenty three dollars for this desk.""" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 6: check Dunky bug + input_text = "For example it normalizes 'medic' into 'm e d i c' or 'yeah' into 'y e a h'." + expected_output = r"""For example it normalizes 'medic' into 'm e d i c' or 'yeah' into 'y e a h'.""" + self.assertEqual(self.normalizer(input_text), expected_output) + + # Test case 7: Time, currency, line-break + input_text = "The alarm went off at 10:00a.m. \nI received $123. It's 12:30pm. I paid $123.45 for this desk." + expected_output = r"""The alarm went off at ten AM. I received one hundred and twenty three dollars. It's twelve thirty PM. I paid one hundred and twenty three dollars forty five cents for this desk.""" + self.assertEqual(self.normalizer(input_text), expected_output) + + def test_normalize2(self): + input_text = r"""The Wizard of Oz: “Lions? And Tigers? And Bears?” + Toy Story: “Buzz, you’re flying!” + As the snake shook its head, a deafening shout behind Harry made both of them jump. ‘DUDLEY! MR DURSLEY! COME AND LOOK AT THIS SNAKE! YOU WON’T BELIEVE WHAT IT’S DOING!’.""" + + expected_output = r"The Wizard of Oz: 'Lions? And Tigers? And Bears?'. Toy Story: 'Buzz, you're flying!'. As the snake shook its head, a deafening shout behind Harry made both of them jump. 'DUDLEY! MR DURSLEY! COME AND LOOK AT THIS SNAKE! YOU WON'T BELIEVE WHAT IT'S DOING!'." + + self.assertEqual(self.normalizer(input_text), expected_output) + + def test_normilize_numbers(self): + input_text = "1234" + result = self.normalizer(input_text) + expected_output = "twelve thirty four" + self.assertEqual(result, expected_output) + + def test_punctuation(self): + input_text = r"""Hello, World! How are you? + Victor — why did you do that? + As the old saying goes, "The early bird catches the worm." (Some people say that the early bird gets the worm.)""" + result = self.normalizer(input_text) + expected_output = r"""Hello, World! How are you?. Victor - why did you do that?. As the old saying goes, 'The early bird catches the worm.' (Some people say that the early bird gets the worm.)""" + self.assertEqual(result, expected_output) + + # Double punctuation + input_text2 = r"""Hello, World!!!! How are you???? + Victor – why did you do that? + As the old saying goes, "The early bird catches the worm." (Some people say that the early bird gets the worm.)""" + result2 = self.normalizer(input_text2) + self.assertEqual(result2, expected_output) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_preprocess_libritts.py b/training/preprocess/tests/test_preprocess_libritts.py new file mode 100644 index 0000000000000000000000000000000000000000..7d6e03fb77f21e41a01cd0b954e60da84f57439f --- /dev/null +++ b/training/preprocess/tests/test_preprocess_libritts.py @@ -0,0 +1,191 @@ +import unittest + +import torch + +from models.config import PreprocessingConfigUnivNet, get_lang_map +from training.preprocess import PreprocessLibriTTS +from training.preprocess.preprocess_libritts import PreprocessForAcousticResult + + +class TestPreprocessLibriTTS(unittest.TestCase): + def setUp(self): + torch.random.manual_seed(42) + lang_map = get_lang_map("en") + processing_lang_type = lang_map.processing_lang_type + self.preprocess_libritts = PreprocessLibriTTS( + PreprocessingConfigUnivNet(processing_lang_type), + ) + + def test_acoustic(self): + # Set the sampling rate and duration of the audio signal + sr_actual = 44100 + duration = 1.0 + + # Set the frequency of the pitch (in Hz) + pitch_freq = 440.0 + + # Generate a time vector for the audio signal + t = torch.linspace(0, duration, int(sr_actual * duration)) + + # Generate a sinusoidal waveform with the specified pitch frequency + audio = torch.sin(2 * torch.pi * pitch_freq * t) + + audio = audio.unsqueeze(0) + + raw_text = "Hello, world!" + + output = self.preprocess_libritts.acoustic( + (audio, sr_actual, raw_text, raw_text, 0, 0, "0"), + ) + + self.assertIsNotNone(output) + + if output is not None: + self.assertIsInstance(output, PreprocessForAcousticResult) + + self.assertIsInstance(output.wav, torch.Tensor) + self.assertIsInstance(output.mel, torch.Tensor) + self.assertIsInstance(output.pitch, torch.Tensor) + self.assertIsInstance(output.phones, torch.Tensor) + self.assertIsInstance(output.raw_text, str) + self.assertIsInstance(output.pitch_is_normalized, bool) + + self.assertEqual(output.wav.shape, torch.Size([22050])) + self.assertEqual(output.mel.shape, torch.Size([100, 86])) + self.assertEqual(output.pitch.shape, torch.Size([86])) + + torch.testing.assert_close( + output.phones, + torch.tensor( + [ + 2.0, + 10.0, + 37.0, + 14.0, + 50.0, + 17.0, + 45.0, + 62.0, + 71.0, + 24.0, + 50.0, + 118.0, + 52.0, + 14.0, + 6.0, + 60.0, + 71.0, + 3.0, + ], + ), + ) + + self.assertEqual(output.raw_text, "Hello, world!") + self.assertFalse(output.pitch_is_normalized) + + def test_acoustic_with_short_audio(self): + audio = torch.randn(1, 22050) + sr_actual = 22050 + raw_text = "Hello, world!" + output = self.preprocess_libritts.acoustic( + (audio, sr_actual, raw_text, raw_text, 0, 0, "0"), + ) + + self.assertIsNone(output) + + def test_acoustic_with_complicated_text(self): + # Set the sampling rate and duration of the audio signal + sr_actual = 44100 + duration = 10.0 + + # Set the frequency of the pitch (in Hz) + pitch_freq = 440.0 + + # Generate a time vector for the audio signal + t = torch.linspace(0, duration, int(sr_actual * duration)) + + # Generate a sinusoidal waveform with the specified pitch frequency + audio = torch.sin(2 * torch.pi * pitch_freq * t).unsqueeze(0) + + raw_text = r"""Hello, world! Wow!!!!! This is amazing????? + It’s a beautiful day… + Mr. Smith paid $111 in U.S.A. on Dec. 17th. We paid $123 for this desk.""" + + output = self.preprocess_libritts.acoustic( + (audio, sr_actual, raw_text, raw_text, 0, 0, "0"), + ) + + self.assertIsNotNone(output) + + if output is not None: + self.assertEqual(output.attn_prior.shape, torch.Size([226, 861])) + self.assertEqual(output.mel.shape, torch.Size([100, 861])) + + self.assertEqual( + output.normalized_text, + "Hello, world! Wow! This is amazing?. It's a beautiful day.. mister Smith paid one hundred and eleven dollars in USA on december seventeenth. We paid one hundred and twenty three dollars for this desk.", + ) + + self.assertEqual(output.phones.shape, torch.Size([226])) + self.assertEqual(len(output.phones_ipa), 224) + + self.assertEqual(output.wav.shape, torch.Size([220500])) + + def test_acoustic_with_long_audio(self): + audio = torch.randn(1, 88200) + sr_actual = 44100 + raw_text = "Hello, world!" + output = self.preprocess_libritts.acoustic( + (audio, sr_actual, raw_text, raw_text, 0, 0, "0"), + ) + + self.assertIsNone(output) + + def test_beta_binomial_prior_distribution(self): + phoneme_count = 10 + mel_count = 20 + prior_dist = self.preprocess_libritts.beta_binomial_prior_distribution( + phoneme_count, + mel_count, + ) + self.assertIsInstance(prior_dist, torch.Tensor) + self.assertEqual(prior_dist.shape, (mel_count, phoneme_count)) + + def test_preprocess_univnet(self): + # Set the sampling rate and duration of the audio signal + sr_actual = 44100 + duration = 10.0 + + # Set the frequency of the pitch (in Hz) + pitch_freq = 440.0 + + # Generate a time vector for the audio signal + t = torch.linspace(0, duration, int(sr_actual * duration)) + + # Generate a sinusoidal waveform with the specified pitch frequency + audio = torch.sin(2 * torch.pi * pitch_freq * t).unsqueeze(0) + + speaker_id = 10 + + output = self.preprocess_libritts.univnet( + (audio, sr_actual, "", "", speaker_id, 0, ""), + ) + + self.assertIsNotNone(output) + + if output is not None: + self.assertIsInstance(output, tuple) + self.assertEqual(len(output), 3) + + mel, audio_segment, speaker_id_output = output + + self.assertIsInstance(mel, torch.Tensor) + self.assertIsInstance(audio_segment, torch.Tensor) + self.assertIsInstance(speaker_id_output, int) + + self.assertEqual(mel.shape, torch.Size([100, 64])) + self.assertEqual(speaker_id_output, speaker_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_tacotron_stft.py b/training/preprocess/tests/test_tacotron_stft.py new file mode 100644 index 0000000000000000000000000000000000000000..48135a6ac7f3c32ecc0169ca3f75b42a8d01a375 --- /dev/null +++ b/training/preprocess/tests/test_tacotron_stft.py @@ -0,0 +1,96 @@ +import unittest + +import torch + +from training.preprocess.tacotron_stft import TacotronSTFT + + +class TestTacotronSTFT(unittest.TestCase): + def setUp(self): + torch.random.manual_seed(0) + + self.batch_size = 2 + self.seq_len = 100 + self.filter_length = 1024 + self.hop_length = 256 + self.win_length = 1024 + self.n_mel_channels = self.filter_length // 2 + 1 + self.sampling_rate = 22050 + self.mel_fmin = 0 + self.mel_fmax = 8000 + self.center = True + + self.model = TacotronSTFT( + filter_length=self.filter_length, + hop_length=self.hop_length, + win_length=self.win_length, + n_mel_channels=self.n_mel_channels, + sampling_rate=self.sampling_rate, + mel_fmin=self.mel_fmin, + mel_fmax=self.mel_fmax, + center=self.center, + ) + + def test_spectrogram(self): + # Test the _spectrogram method + y = torch.randn(self.batch_size, self.filter_length // 2) + y = y / torch.max(torch.abs(y)) + + spec = self.model._spectrogram(y) + + self.assertEqual( + spec.shape, + (self.batch_size, self.filter_length // 2 + 1, 6, self.batch_size), + ) + + def test_linear_spectrogram(self): + # Test the linear_spectrogram method + y = torch.randn(self.batch_size, self.filter_length // 2) + y = y / torch.max(torch.abs(y)) + + spec = self.model.linear_spectrogram(y) + + self.assertEqual(spec.shape, (self.batch_size, self.filter_length // 2 + 1, 6)) + + def test_forward(self): + # Test the forward method + y = torch.randn(self.n_mel_channels, self.filter_length // 2) + y = y / torch.max(torch.abs(y)) + + spec, mel = self.model(y) + + self.assertEqual( + spec.shape, + (self.filter_length // 2 + 1, self.filter_length // 2 + 1, 6), + ) + self.assertEqual( + mel.shape, + (self.filter_length // 2 + 1, self.filter_length // 2 + 1, 6), + ) + + def test_spectral_normalize_torch(self): + # Test the spectral_normalize_torch method + magnitudes = torch.randn(self.batch_size, self.n_mel_channels, self.seq_len) + output = self.model.spectral_normalize_torch(magnitudes) + self.assertEqual( + output.shape, (self.batch_size, self.n_mel_channels, self.seq_len), + ) + + def test_dynamic_range_compression_torch(self): + # Test the dynamic_range_compression_torch method + x = torch.randn(self.batch_size, self.n_mel_channels, self.seq_len) + output = self.model.dynamic_range_compression_torch(x) + self.assertEqual( + output.shape, (self.batch_size, self.n_mel_channels, self.seq_len), + ) + + def test_get_mel_from_wav(self): + # Test the get_mel_from_wav method + audio = torch.randn(44100) + audio /= torch.max(torch.abs(audio)) + melspec = self.model.get_mel_from_wav(audio) + self.assertEqual(melspec.shape, (self.n_mel_channels, 176)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_tokenizer_ipa.py b/training/preprocess/tests/test_tokenizer_ipa.py new file mode 100644 index 0000000000000000000000000000000000000000..da4917b0aee1456abf8f8f44d907ab99c6f83f8e --- /dev/null +++ b/training/preprocess/tests/test_tokenizer_ipa.py @@ -0,0 +1,34 @@ +import unittest + +from training.preprocess.tokenizer_ipa import TokenizerIPA + + +class TestTokenizerIPA(unittest.TestCase): + def setUp(self): + self.tokenizer = TokenizerIPA() + + def test_init(self): + self.assertEqual(self.tokenizer.lang, "en_us") + self.assertIsNotNone(self.tokenizer.phonemizer) + self.assertIsNotNone(self.tokenizer.tokenizer) + + def test_call(self): + text = "hello world" + phones_ipa, tokens = self.tokenizer(text) + + self.assertIsInstance(phones_ipa, str) + self.assertIsInstance(tokens, list) + self.assertTrue(all(isinstance(token, int) for token in tokens)) + + def test_call_with_punctuation(self): + text = "hello world" + phones_ipa, tokens = self.tokenizer(text) + + text2 = "Hello, world!" + phones_ipa2, tokens2 = self.tokenizer(text2) + + self.assertNotEqual(phones_ipa, phones_ipa2) + self.assertNotEqual(tokens, tokens2) + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_tokenizer_ipa_espeak.py b/training/preprocess/tests/test_tokenizer_ipa_espeak.py new file mode 100644 index 0000000000000000000000000000000000000000..48af33eb4e4bde620a370464f4a1314226f38ab7 --- /dev/null +++ b/training/preprocess/tests/test_tokenizer_ipa_espeak.py @@ -0,0 +1,47 @@ +import unittest + +from training.preprocess.tokenizer_ipa import TokenizerIPA +from training.preprocess.tokenizer_ipa_espeak import TokenizerIpaEspeak + + +class TestTokenizerIPA(unittest.TestCase): + def setUp(self): + self.tokenizerIPA = TokenizerIPA() + self.tokenizerIpaEspeak = TokenizerIpaEspeak() + + def test_init(self): + self.assertEqual(self.tokenizerIPA.lang, "en_us") + self.assertIsNotNone(self.tokenizerIPA.phonemizer) + self.assertIsNotNone(self.tokenizerIPA.tokenizer) + + def test_call(self): + text = "'DUDLEY! MR DURSLEY! COME AND LOOK AT THIS SNAKE! YOU WON'T BELIEVE WHAT IT'S DOING!'Dudley came waddling towards them as fast as he could." + # phones_ipa, tokens = self.tokenizerIPA(text) + phones_ipa_espeak, tokens_espeak = self.tokenizerIpaEspeak(text) + + self.assertIsInstance(phones_ipa_espeak, str) + self.assertIsInstance(tokens_espeak, list) + self.assertTrue(all(isinstance(token, int) for token in tokens_espeak)) + + def test_case_sensitive(self): + text = "WHAT IT'S DOING! Dudley came waddling towards them as fast as he could." + # phones_ipa, tokens = self.tokenizerIPA(text) + phones_ipa_espeak, tokens_espeak = self.tokenizerIpaEspeak(text) + + self.assertIsInstance(phones_ipa_espeak, str) + self.assertIsInstance(tokens_espeak, list) + self.assertTrue(all(isinstance(token, int) for token in tokens_espeak)) + + def test_call_with_punctuation(self): + text = "hello world" + phones_ipa, tokens = self.tokenizerIpaEspeak(text) + + text2 = "Hello, world!" + phones_ipa2, tokens2 = self.tokenizerIpaEspeak(text2) + + self.assertNotEqual(phones_ipa, phones_ipa2) + self.assertNotEqual(tokens, tokens2) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tests/test_wav2vec_aligner.py b/training/preprocess/tests/test_wav2vec_aligner.py new file mode 100644 index 0000000000000000000000000000000000000000..cc1b2f4bcc618ae8000d1470862a4ffdd80acb0e --- /dev/null +++ b/training/preprocess/tests/test_wav2vec_aligner.py @@ -0,0 +1,211 @@ +import unittest + +import torch + +from training.preprocess.wav2vec_aligner import Wav2VecAligner + + +class TestWav2VecAligner(unittest.TestCase): + def setUp(self): + self.model = Wav2VecAligner() + self.text = "I HAD THAT CURIOSITY BESIDE ME AT THIS MOMENT" + self.wav_path = "./mocks/audio_example.wav" + + def test_load_audio(self): + _, sample_rate = self.model.load_audio(self.wav_path) + + self.assertEqual(sample_rate, 16_000) + + with self.assertRaises(FileNotFoundError): + self.model.load_audio("./nonexistent/path.wav") + + def test_encode(self): + tokens = self.model.encode(self.text) + + torch.testing.assert_close( + tokens, + torch.tensor( + [ + [ + 10, + 4, + 11, + 7, + 14, + 4, + 6, + 11, + 7, + 6, + 4, + 19, + 16, + 13, + 10, + 8, + 12, + 10, + 6, + 22, + 4, + 24, + 5, + 12, + 10, + 14, + 5, + 4, + 17, + 5, + 4, + 7, + 6, + 4, + 6, + 11, + 10, + 12, + 4, + 17, + 8, + 17, + 5, + 9, + 6, + ], + ], + ), + ) + + def test_decode(self): + transcript = self.model.decode( + [ + [ + 10, + 4, + 11, + 7, + 14, + 4, + 6, + 11, + 7, + 6, + 4, + 19, + 16, + 13, + 10, + 8, + 12, + 10, + 6, + 22, + 4, + 24, + 5, + 12, + 10, + 14, + 5, + 4, + 17, + 5, + 4, + 7, + 6, + 4, + 6, + 11, + 10, + 12, + 4, + 17, + 8, + 17, + 5, + 9, + 6, + ], + ], + ) + + self.assertEqual(transcript, self.text) + + def test_align_single_sample(self): + audio_input, _ = self.model.load_audio(self.wav_path) + emissions, tokens, transcript = self.model.align_single_sample( + audio_input, self.text, + ) + + self.assertEqual(emissions.shape, torch.Size([169, 32])) + + self.assertEqual( + len(tokens), + 47, + ) + + self.assertEqual(transcript, "|I|HAD|THAT|CURIOSITY|BESIDE|ME|AT|THIS|MOMENT|") + + def test_get_trellis(self): + audio_input, _ = self.model.load_audio(self.wav_path) + emissions, tokens, _ = self.model.align_single_sample(audio_input, self.text) + trellis = self.model.get_trellis(emissions, tokens) + + self.assertEqual(emissions.shape, torch.Size([169, 32])) + self.assertEqual(len(tokens), 47) + + # Add assertions here based on the expected behavior of get_trellis + self.assertIsInstance(trellis, torch.Tensor) + self.assertEqual(trellis.shape, torch.Size([169, 47])) + + def test_backtrack(self): + audio_input, _ = self.model.load_audio(self.wav_path) + emissions, tokens, _ = self.model.align_single_sample(audio_input, self.text) + trellis = self.model.get_trellis(emissions, tokens) + path = self.model.backtrack(trellis, emissions, tokens) + + # Add assertions here based on the expected behavior of backtrack + self.assertIsInstance(path, list) + self.assertEqual(len(path), 169) + + def test_merge_repeats(self): + audio_input, _ = self.model.load_audio(self.wav_path) + emissions, tokens, transcript = self.model.align_single_sample( + audio_input, self.text, + ) + trellis = self.model.get_trellis(emissions, tokens) + path = self.model.backtrack(trellis, emissions, tokens) + merged_path = self.model.merge_repeats(path, transcript) + + # Add assertions here based on the expected behavior of merge_repeats + self.assertIsInstance(merged_path, list) + self.assertEqual(len(merged_path), 47) + + def test_merge_words(self): + audio_input, _ = self.model.load_audio(self.wav_path) + emissions, tokens, transcript = self.model.align_single_sample( + audio_input, self.text, + ) + trellis = self.model.get_trellis(emissions, tokens) + path = self.model.backtrack(trellis, emissions, tokens) + merged_path = self.model.merge_repeats(path, transcript) + merged_words = self.model.merge_words(merged_path) + + # Add assertions here based on the expected behavior of merge_words + self.assertIsInstance(merged_words, list) + self.assertEqual(len(merged_words), 9) + + def test_forward(self): + result = self.model(self.wav_path, self.text) + + # self.assertEqual(result, expected_result) + self.assertEqual(len(result), 9) + + def test_save_segments(self): + # self.model.save_segments(self.wav_path, self.text, "./mocks/wav2vec_aligner/audio") + self.assertEqual(True, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/preprocess/tokenizer_ipa.py b/training/preprocess/tokenizer_ipa.py new file mode 100644 index 0000000000000000000000000000000000000000..90fd34b3ed6233bcab3d02ab68fcdbece6fa44f3 --- /dev/null +++ b/training/preprocess/tokenizer_ipa.py @@ -0,0 +1,139 @@ +from typing import List, Tuple, Union + +from dp.phonemizer import Phonemizer +from dp.preprocessing.text import SequenceTokenizer + +from models.config import get_lang_map + + +class TokenizerIPA: + r"""DEPRECATED: TokenizerIPA is a class for tokenizing International Phonetic Alphabet (IPA) phonemes. + + Attributes: + lang (str): Language to be used. Default is "en". + phonemizer_checkpoint (str): Path to the phonemizer checkpoint file. + phonemizer (Phonemizer): Phonemizer object for converting text to phonemes. + tokenizer (SequenceTokenizer): SequenceTokenizer object for tokenizing the phonemes. + """ + + def __init__( + self, + lang: str = "en", + phonemizer_checkpoint: str = "../checkpoints/en_us_cmudict_ipa_forward.pt", + ): + r"""Initializes TokenizerIPA with the given language and phonemizer checkpoint. + + Args: + lang (str): The language to be used. Default is "en". + phonemizer_checkpoint (str): The path to the phonemizer checkpoint file. + """ + lang_map = get_lang_map(lang) + self.lang = lang_map.phonemizer + + self.phonemizer = Phonemizer.from_checkpoint(phonemizer_checkpoint) + + phoneme_symbols = [ + # IPA symbols + "a", + "b", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "æ", + "ç", + "ð", + "ø", + "ŋ", + "œ", + "ɐ", + "ɑ", + "ɔ", + "ə", + "ɛ", + "ɝ", + "ɹ", + "ɡ", + "ɪ", + "ʁ", + "ʃ", + "ʊ", + "ʌ", + "ʏ", + "ʒ", + "ʔ", + "ˈ", + "ˌ", + "ː", + "̃", + "̍", + "̥", + "̩", + "̯", + "͡", + "θ", + # Punctuation + "!", + "?", + ",", + ".", + "-", + ":", + ";", + '"', + "'", + "(", + ")", + " ", + ] + + self.tokenizer = SequenceTokenizer( + phoneme_symbols, + languages=["de", "en_us"], + lowercase=True, + char_repeats=1, + append_start_end=True, + ) + + # test_text = "Hello, World!" + # print("Initializing TokenizerIPA, check on: ", test_text) + + # phones_ipa = self.phonemizer(test_text, lang=self.lang) + # tokens = self.tokenizer(phones_ipa, language=self.lang) + + # print("phones_ipa: ", phones_ipa) + # print("tokens: ", tokens) + # print("decoded: ", self.tokenizer.decode(tokens)) + + def __call__(self, text: str) -> Tuple[Union[str, List[str]], List[int]]: + r"""Converts the input text to phonemes and tokenizes them. + + Args: + text (str): The input text to be tokenized. + + Returns: + list: The tokenized phonemes. + + """ + phones_ipa = self.phonemizer(text, lang=self.lang) + tokens = self.tokenizer(phones_ipa, language=self.lang) + + return phones_ipa, tokens diff --git a/training/preprocess/tokenizer_ipa_espeak.py b/training/preprocess/tokenizer_ipa_espeak.py new file mode 100644 index 0000000000000000000000000000000000000000..6164111d3db4020bc4b7eaf755e3d5f5f190e29f --- /dev/null +++ b/training/preprocess/tokenizer_ipa_espeak.py @@ -0,0 +1,65 @@ +from logging import ERROR, Logger +import os + +from phonemizer.backend import EspeakBackend + +# IPA Phonemizer: https://github.com/bootphon/phonemizer +from phonemizer.backend.espeak.wrapper import EspeakWrapper + +# Create a Logger instance +logger = Logger("my_logger") +# Set the level to ERROR +logger.setLevel(ERROR) + +from dp.preprocessing.text import SequenceTokenizer + +from models.config import get_lang_map +from models.config.symbols import phones + +# INFO: Fix for windows, used for local env +if os.name == "nt": + ESPEAK_LIBRARY = os.getenv( + "ESPEAK_LIBRARY", + "C:\\Program Files\\eSpeak NG\\libespeak-ng.dll", + ) + EspeakWrapper.set_library(ESPEAK_LIBRARY) + + +class TokenizerIpaEspeak: + def __init__(self, lang: str = "en"): + lang_map = get_lang_map(lang) + self.lang = lang_map.phonemizer_espeak + self.lang_seq = lang_map.phonemizer + + # NOTE: for backward compatibility with previous IPA tokenizer see the TokenizerIPA class + self.tokenizer = SequenceTokenizer( + phones, + languages=["de", "en_us"], + lowercase=True, + char_repeats=1, + append_start_end=True, + ) + + self.phonemizer = EspeakBackend( + language=self.lang, + preserve_punctuation=True, + with_stress=True, + words_mismatch="ignore", + logger=logger, + ).phonemize + + def __call__(self, text: str): + r"""Converts the input text to phonemes and tokenizes them. + + Args: + text (str): The input text to be tokenized. + + Returns: + Tuple[Union[str, List[str]], List[int]]: IPA phonemes and tokens. + + """ + phones_ipa = "".join(self.phonemizer([text])) + + tokens = self.tokenizer(phones_ipa, language=self.lang_seq) + + return phones_ipa, tokens diff --git a/training/preprocess/wav2vec_aligner.py b/training/preprocess/wav2vec_aligner.py new file mode 100644 index 0000000000000000000000000000000000000000..797118b803cd68c9370225d9c6c31ee5eb85a15b --- /dev/null +++ b/training/preprocess/wav2vec_aligner.py @@ -0,0 +1,387 @@ +from dataclasses import dataclass +import os +from typing import List, Tuple + +import torch +from torch.nn import Module +import torchaudio +from transformers import ( + AutoConfig, + AutoModelForCTC, + AutoProcessor, +) + + +@dataclass +class Item: + r"""A data class that represents an item with a sentence, + a path to a wav file, and an output path. + """ + + sent: str + wav_path: str + out_path: str + + +@dataclass +class Point: + r"""A data class that represents a point with a token index, + a time index, and a score. + """ + + token_index: int + time_index: int + score: float + + +@dataclass +class Segment: + r"""A data class that represents a segment with a label, + a start time, an end time, a duration, and a score. + """ + + label: str + start: int + end: int + # TODO: check that the scale of duration is correct... + duration: float + score: float + + @property + def length(self): + return self.end - self.start + + +class Wav2VecAligner(Module): + r"""Wav2VecAligner model. + + The Wav2VecAligner model is designed for aligning audio data with text data. + This class handles the training and validation of the Wav2VecAligner model. + + Attributes + config (AutoConfig): The configuration for the pre-trained model. + model (AutoModelForCTC): The pre-trained model. + processor (AutoProcessor): The processor for the pre-trained model. + labels (List): The labels from the vocabulary of the tokenizer. + blank_id (int): The ID of the blank token. + + Methods + load_audio: Load an audio file from the specified path. + encode: Encode the labels. + decode: Decode the tokens. + align_single_sample: Align a single sample of audio data with the corresponding text. + get_trellis: Build a trellis matrix that represents the probabilities of each source token being at a certain time step. + backtrack: Walk backwards from the last + merge_repeats: Merge repeated tokens into a single segment. + merge_words: Merge words in the given path. + forward: Perform the forward pass of the model, which involves loading the audio data, aligning the audio with the text, building the trellis, backtracking to find the optimal path, merging repeated tokens, and finally merging words. + + """ + + def __init__( + self, + model_name: str = "facebook/wav2vec2-base-960h", + ): + r"""Initialize a new instance of the Wav2VecAligner class. + + Args: + model_name (str): The name of the pre-trained model to use. Defaults to "facebook/wav2vec2-base-960h". + """ + super().__init__() + + # Load the config + self.config = AutoConfig.from_pretrained(model_name) + + self.model = AutoModelForCTC.from_pretrained(model_name) + self.model.eval() + + self.processor = AutoProcessor.from_pretrained(model_name) + + # get labels from vocab + self.labels = list(self.processor.tokenizer.get_vocab().keys()) + + for i in range(len(self.labels)): + if self.labels[i] == "[PAD]" or self.labels[i] == "": + self.blank_id = i + + print("Blank Token id [PAD]/", self.blank_id) + + def load_audio(self, wav_path: str) -> Tuple[torch.Tensor, int]: + r"""Load an audio file from the specified path. + + Args: + wav_path (str): The path to the audio file. + + Returns: + Tuple[torch.Tensor, int]: A tuple containing the loaded audio data and the sample rate, or a FileNotFoundError if the file does not exist. + """ + if not os.path.isfile(wav_path): + raise FileNotFoundError(wav_path, "Not found in wavs directory") + + speech_array, sampling_rate = torchaudio.load(wav_path) # type: ignore + return speech_array, sampling_rate + + def encode(self, text: str) -> List: + # encode labels + with self.processor.as_target_processor(): + return self.processor(text, return_tensors="pt").input_ids + + def decode(self, tokens: List): + # Decode tokens + decoded = self.processor.batch_decode(tokens) + return decoded[0] + + def align_single_sample( + self, audio_input: torch.Tensor, text: str, + ) -> Tuple[torch.Tensor, List, str]: + r"""Align a single sample of audio data with the corresponding text. + + Args: + audio_input (torch.Tensor): The audio data. + text (str): The corresponding text. + + Returns: + Tuple[torch.Tensor, List, str]: A tuple containing the emissions, the tokens, and the transcript. + """ + transcript = "|".join(text.split(" ")) + transcript = f"|{transcript}|" + + with torch.inference_mode(): + logits = self.model(audio_input).logits + + # Get the emission probability at frame level + # Compute the probability in log-domain to avoid numerical instability + # For this purpose, we normalize the emission with `torch.log_softmax()` + emissions = torch.log_softmax(logits, dim=-1) + emissions = emissions[0] + + tokens = self.encode(transcript)[0] + + return emissions, tokens, transcript + + def get_trellis( + self, + emission: torch.Tensor, + tokens: List, + ) -> torch.Tensor: + r"""Build a trellis matrix of shape (num_frames + 1, num_tokens + 1) + that represents the probabilities of each source token being at a certain time step. + + Since we are looking for the most likely transitions, we take the more likely path for the value of $k_{(t+1,j+1)}$, that is: + + $k_{t+1, j+1} = \max(k_{t, j} p_{t+1, c_{j+1}}, k_{t, j+1} p_{t+1, \text{repeat}})$ + + Args: + emission (torch.Tensor): The emission tensor. + tokens (List): The list of tokens. + + Returns: + torch.Tensor: The trellis matrix. + """ + num_frames = emission.size(0) + num_tokens = len(tokens) + + # Trellis has extra diemsions for both time axis and tokens. + # The extra dim for tokens represents (start-of-sentence) + # The extra dim for time axis is for simplification of the code. + + trellis = torch.zeros((num_frames, num_tokens)) + trellis[1:, 0] = torch.cumsum(emission[1:, self.blank_id], 0) + trellis[0, 1:] = -float("inf") + trellis[-num_tokens + 1 :, 0] = float("inf") + + for t in range(num_frames - 1): + trellis[t + 1, 1:] = torch.maximum( + # Score for staying at the same token + trellis[t, 1:] + emission[t, self.blank_id], + # Score for changing to the next token + trellis[t, :-1] + emission[t, tokens[1:]], + ) + return trellis + + def backtrack( + self, + trellis: torch.Tensor, + emission: torch.Tensor, + tokens: List, + ) -> List[Point]: + r"""Walk backwards from the last (sentence_token, time_step) pair to build the optimal sequence alignment path. + + Args: + trellis (torch.Tensor): The trellis matrix. + emission (torch.Tensor): The emission tensor. + tokens (List): The list of tokens. + + Returns: + List[Point]: The optimal sequence alignment path. + """ + # Note: + # j and t are indices for trellis, which has extra dimensions + # for time and tokens at the beginning. + # When referring to time frame index `T` in trellis, + # the corresponding index in emission is `T-1`. + # Similarly, when referring to token index `J` in trellis, + # the corresponding index in transcript is `J-1`. + t, j = trellis.size(0) - 1, trellis.size(1) - 1 + + path = [Point(j, t, emission[t, self.blank_id].exp().item())] + while j > 0: + # Should not happen but just in case + assert t > 0 + + # 1. Figure out if the current position was stay or change + # Frame-wise score of stay vs change + p_stay = emission[t - 1, self.blank_id] + p_change = emission[t - 1, tokens[j]] + + # Context-aware score for stay vs change + stayed = trellis[t - 1, j] + p_stay + changed = trellis[t - 1, j - 1] + p_change + + # Update position + t -= 1 + if changed > stayed: + j -= 1 + + # Store the path with frame-wise probability. + prob = (p_change if changed > stayed else p_stay).exp().item() + path.append(Point(j, t, prob)) + + # Now j == 0, which means, it reached the SoS. + # Fill up the rest for the sake of visualization + while t > 0: + prob = emission[t - 1, self.blank_id].exp().item() + path.append(Point(j, t - 1, prob)) + t -= 1 + + return path[::-1] + + def merge_repeats(self, path: List[Point], transcript: str) -> List[Segment]: + r"""Merge repeated tokens into a single segment. + + Args: + path (List[Point]): The sequence alignment path. + transcript (str): The transcript. + + Returns: + List[Segment]: The list of segments. + + Note: this shouldn't affect repeated characters from the + original sentences (e.g. `ll` in `hello`) + """ + i1, i2 = 0, 0 + segments = [] + while i1 < len(path): + while i2 < len(path) and path[i1].token_index == path[i2].token_index: + i2 += 1 + score = sum(path[k].score for k in range(i1, i2)) / (i2 - i1) + + x0, x1 = path[i1].time_index, path[i2 - 1].time_index + 1 + duration = x1 - x0 + + segments.append( + Segment( + transcript[path[i1].token_index], + x0, + x1, + duration, + score, + ), + ) + i1 = i2 + return segments + + # Merge words + def merge_words( + self, segments: List[Segment], separator: str = "|", + ) -> List[Segment]: + r"""Merge words in the given path. + + Args: + segments (List[Segment]): The list of segments. + separator (str): The separator character. Defaults to "|". + + Returns: + List[Segment]: The list of merged words. + """ + words = [] + i1, i2 = 0, 0 + while i1 < len(segments): + if i2 >= len(segments) or segments[i2].label == separator: + if i1 != i2: + segs = segments[i1:i2] + word = "".join([seg.label for seg in segs]) + score = sum(seg.score * seg.length for seg in segs) / sum( + seg.length for seg in segs + ) + + x0, x1 = segments[i1].start, segments[i2 - 1].end + duration = x1 - x0 + + words.append(Segment(word, x0, x1, duration, score)) + i1 = i2 + 1 + i2 = i1 + else: + i2 += 1 + return words + + def forward(self, wav_path: str, text: str) -> List[Segment]: + r"""Perform the forward pass of the model, which involves loading the audio data, aligning the audio with the text, + building the trellis, backtracking to find the optimal path, merging repeated tokens, and finally merging words. + + Args: + wav_path (str): The path to the audio file. + text (str): The corresponding text. + + Returns: + List[Segment]: The list of segments representing the alignment of the audio data with the text. + """ + audio_input, _ = self.load_audio(wav_path) + + emissions, tokens, transcript = self.align_single_sample(audio_input, text) + + trellis = self.get_trellis(emissions, tokens) + + path = self.backtrack(trellis, emissions, tokens) + + merged_path = self.merge_repeats(path, transcript) + + return self.merge_words(merged_path) + + def save_segments(self, wav_path: str, text: str, save_dir: str): + r"""Perform the forward pass of the model to get the segments and save each segment to a file. + Used for debugging purposes. + + Args: + wav_path (str): The path to the audio file. + text (str): The corresponding text. + save_dir (str): The directory where the audio files should be saved. + + Returns: + None + """ + word_segments = self.forward(wav_path, text) + + waveform, sampling_rate = self.load_audio(wav_path) + + emissions, tokens, _ = self.align_single_sample(waveform, text) + + trellis = self.get_trellis(emissions, tokens) + + ratio = waveform.size(1) / trellis.size(0) + + for i, word in enumerate(word_segments): + x0 = int(ratio * word.start) + x1 = int(ratio * word.end) + + print( + f"{word.label} ({word.score:.2f}): {x0 / sampling_rate:.3f} - {x1 / sampling_rate:.3f} sec", + ) + + segment_waveform = waveform[:, x0:x1] + + # Save the segment waveform to a file + filename = f"{i}_{word.label}.wav" + torchaudio.save( # type: ignore + os.path.join(save_dir, filename), segment_waveform, sampling_rate, + ) diff --git a/training/tests/__init__.py b/training/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/training/tests/test_np_tools.py b/training/tests/test_np_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..16f8f6c174f019a7fdb3a1927247ddd6ea67a0d2 --- /dev/null +++ b/training/tests/test_np_tools.py @@ -0,0 +1,91 @@ +import unittest + +import numpy as np + +from training.np_tools import pad_1D, pad_2D, pad_3D + + +class TestPad(unittest.TestCase): + def test_pad_1D(self): + # Test case 1: Pad a list of 1D numpy arrays with different lengths + inputs = [np.array([1, 2, 3]), np.array([4, 5]), np.array([6])] + expected_output = np.array([[1, 2, 3], [4, 5, 0], [6, 0, 0]]) + self.assertTrue(np.array_equal(pad_1D(inputs), expected_output)) + + # Test case 2: Pad a list of 1D numpy arrays with the same length + inputs = [np.array([1, 2, 3]), np.array([4, 5, 6]), np.array([7, 8, 9])] + expected_output = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + self.assertTrue(np.array_equal(pad_1D(inputs), expected_output)) + + # Test case 3: Pad a list of 1D numpy arrays with a non-zero pad value + inputs = [np.array([1, 2]), np.array([3, 4, 5]), np.array([6, 7, 8, 9])] + expected_output = np.array([[1, 2, 0, 0], [3, 4, 5, 0], [6, 7, 8, 9]]) + self.assertTrue(np.array_equal(pad_1D(inputs, pad_value=0.0), expected_output)) + + # Test case 4: Pad a list of 1D numpy arrays with a non-zero pad value + inputs = [np.array([1, 2]), np.array([3, 4, 5]), np.array([6, 7, 8, 9])] + expected_output = np.array([[1, 2, 1, 1], [3, 4, 5, 1], [6, 7, 8, 9]]) + self.assertTrue(np.array_equal(pad_1D(inputs, pad_value=1.0), expected_output)) + + # Test case 5: Pad a list of 1D numpy arrays with a single non-empty array + inputs = [np.array([1, 2, 3])] + expected_output = np.array([[1, 2, 3]]) + self.assertTrue(np.array_equal(pad_1D(inputs), expected_output)) + + def test_pad_2D(self): + # Test case 1: Pad a list of 2D numpy arrays with different shapes + inputs = [np.array([[1, 2], [3, 4]]), np.array([[5, 6, 7], [8, 9, 10]])] + expected_output = np.array([[[1, 2, 0], [3, 4, 0]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(np.array_equal(pad_2D(inputs), expected_output)) + + # Test case 2: Pad a list of 2D numpy arrays with the same shape + inputs = [np.array([[1, 2], [3, 4]]), np.array([[5, 6], [7, 8]])] + expected_output = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + self.assertTrue(np.array_equal(pad_2D(inputs), expected_output)) + + # Test case 3: Pad a list of 2D numpy arrays with a non-zero pad value + inputs = [np.array([[1, 2], [3, 4]]), np.array([[5, 6, 7], [8, 9, 10]])] + expected_output = np.array([[[1, 2, 1], [3, 4, 1]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(np.array_equal(pad_2D(inputs, pad_value=1.0), expected_output)) + + # Test case 4: Pad a list of 2D numpy arrays with a maximum length + inputs = [np.array([[1, 2], [3, 4]]), np.array([[5, 6, 7], [8, 9, 10]])] + expected_output = np.array([[[1, 2, 0], [3, 4, 0]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(np.array_equal(pad_2D(inputs, maxlen=3), expected_output)) + + # Test case 5: Pad a list of 2D numpy arrays with a single non-empty array + inputs = [np.array([[1, 2], [3, 4]])] + expected_output = np.array([[[1, 2], [3, 4]]]) + self.assertTrue(np.array_equal(pad_2D(inputs), expected_output)) + + def test_pad_3D(self): + # Test case 1: Pad a 3D numpy array with different dimensions + inputs = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]]) + expected_output = np.array( + [ + [[1, 2, 0], [3, 4, 0], [0, 0, 0]], + [[5, 6, 0], [7, 8, 0], [0, 0, 0]], + [[9, 10, 0], [11, 12, 0], [0, 0, 0]], + ], + ) + self.assertTrue(np.array_equal(pad_3D(inputs, B=3, T=3, L=3), expected_output)) + + # Test case 2: Pad a 3D numpy array with the same dimensions + inputs = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]]) + expected_output = np.array( + [ + [[1, 2], [3, 4]], + [[5, 6], [7, 8]], + [[9, 10], [11, 12]], + ], + ) + self.assertTrue(np.array_equal(pad_3D(inputs, B=3, T=2, L=2), expected_output)) + + # Test case 3: Pad a 3D numpy array with a single element + inputs = np.array([[[1, 2], [3, 4]]]) + expected_output = np.array([[[1, 2, 0], [3, 4, 0]]]) + self.assertTrue(np.array_equal(pad_3D(inputs, B=1, T=2, L=3), expected_output)) + + +if __name__ == "__main__": + unittest.main() diff --git a/training/tests/test_tools.py b/training/tests/test_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..daa89dcfe2cc687166f022873910571be381c65c --- /dev/null +++ b/training/tests/test_tools.py @@ -0,0 +1,106 @@ +import unittest + +import numpy as np +import torch + +from training.tools import pad_1D, pad_2D, pad_3D + + +class TestPad(unittest.TestCase): + def test_pad_1D(self): + # Test case 1: Pad a list of 1D numpy arrays with different lengths + inputs = [torch.tensor([1, 2, 3]), torch.tensor([4, 5]), torch.tensor([6])] + expected_output = torch.tensor([[1, 2, 3], [4, 5, 0], [6, 0, 0]]) + self.assertTrue(torch.allclose(pad_1D(inputs), expected_output)) + + # Test case 2: Pad a list of 1D numpy arrays with the same length + inputs = [torch.tensor([1, 2, 3]), torch.tensor([4, 5, 6]), torch.tensor([7, 8, 9])] + expected_output = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + self.assertTrue(torch.allclose(pad_1D(inputs), expected_output)) + + # Test case 3: Pad a list of 1D numpy arrays with a non-zero pad value + inputs = [torch.tensor([1, 2]), torch.tensor([3, 4, 5]), torch.tensor([6, 7, 8, 9])] + expected_output = torch.tensor([[1, 2, 0, 0], [3, 4, 5, 0], [6, 7, 8, 9]]) + self.assertTrue(torch.allclose(pad_1D(inputs, pad_value=0.0), expected_output)) + + # Test case 4: Pad a list of 1D numpy arrays with a non-zero pad value + inputs = [torch.tensor([1, 2]), torch.tensor([3, 4, 5]), torch.tensor([6, 7, 8, 9])] + expected_output = torch.tensor([[1, 2, 1, 1], [3, 4, 5, 1], [6, 7, 8, 9]]) + self.assertTrue(torch.allclose(pad_1D(inputs, pad_value=1.0), expected_output)) + + # Test case 5: Pad a list of 1D numpy arrays with a single non-empty array + inputs = [torch.tensor([1, 2, 3])] + expected_output = torch.tensor([[1, 2, 3]]) + self.assertTrue(torch.allclose(pad_1D(inputs), expected_output)) + + def test_pad_2D(self): + # Test case 1: Pad a list of 2D numpy arrays with different shapes + inputs = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6, 7], [8, 9, 10]])] + expected_output = torch.tensor([[[1, 2, 0], [3, 4, 0]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(torch.allclose(pad_2D(inputs), expected_output)) + + # Test case 2: Pad a list of 2D numpy arrays with the same shape + inputs = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6], [7, 8]])] + expected_output = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + self.assertTrue(torch.allclose(pad_2D(inputs), expected_output)) + + # Test case 3: Pad a list of 2D numpy arrays with a non-zero pad value + inputs = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6, 7], [8, 9, 10]])] + expected_output = torch.tensor([[[1, 2, 1], [3, 4, 1]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(torch.allclose(pad_2D(inputs, pad_value=1.0), expected_output)) + + # Test case 4: Pad a list of 2D numpy arrays with a maximum length + inputs = [torch.tensor([[1, 2], [3, 4]]), torch.tensor([[5, 6, 7], [8, 9, 10]])] + expected_output = torch.tensor([[[1, 2, 0], [3, 4, 0]], [[5, 6, 7], [8, 9, 10]]]) + self.assertTrue(torch.allclose(pad_2D(inputs, maxlen=3), expected_output)) + + # Test case 5: Pad a list of 2D numpy arrays with a single non-empty array + inputs = [torch.tensor([[1, 2], [3, 4]])] + expected_output = torch.tensor([[[1, 2], [3, 4]]]) + self.assertTrue(torch.allclose(pad_2D(inputs), expected_output)) + + def test_pad_3D(self): + # Test case 1: Pad a 3D numpy array with different dimensions + inputs = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]]) + expected_output = torch.tensor( + [ + [[1, 2, 0], [3, 4, 0], [0, 0, 0]], + [[5, 6, 0], [7, 8, 0], [0, 0, 0]], + [[9, 10, 0], [11, 12, 0], [0, 0, 0]], + ], + ) + self.assertTrue(torch.allclose(pad_3D(inputs, B=3, T=3, L=3), expected_output)) + + # Test case 2: Pad a 3D numpy array with the same dimensions + inputs = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]]) + expected_output = torch.tensor( + [ + [[1, 2], [3, 4]], + [[5, 6], [7, 8]], + [[9, 10], [11, 12]], + ], + ) + self.assertTrue(torch.allclose(pad_3D(inputs, B=3, T=2, L=2), expected_output)) + + # Test case 3: Pad a 3D numpy array with a single element + inputs = torch.tensor([[[1, 2], [3, 4]]]) + expected_output = torch.tensor([[[1, 2, 0], [3, 4, 0]]]) + self.assertTrue(torch.allclose(pad_3D(inputs, B=1, T=2, L=3), expected_output)) + + # Test case: Pad a list of 3D numpy arrays with different dimensions + inputs = [ + torch.tensor([[1, 2], [3, 4]]), + torch.tensor([[5, 6], [7, 8], [9, 10]]), + torch.tensor([[11, 12], [13, 14], [15, 16]]), + ] + expected_output = torch.tensor( + [ + [[1, 2, 0], [3, 4, 0], [0, 0, 0], [0, 0, 0]], + [[5, 6, 0], [7, 8, 0], [9, 10, 0], [0, 0, 0]], + [[11, 12, 0], [13, 14, 0], [15, 16, 0], [0, 0, 0]], + ], + ) + self.assertTrue(torch.allclose(pad_3D(inputs, B=3, T=4, L=3), expected_output)) + +if __name__ == "__main__": + unittest.main() diff --git a/training/tools.py b/training/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..c03816d426cc3b77d09d0c144ebd8e3fe49b8919 --- /dev/null +++ b/training/tools.py @@ -0,0 +1,62 @@ +from typing import List, Union + +import torch +from torch import Tensor, nn + + +def pad_1D(inputs: List[Tensor], pad_value: float = 0.0) -> Tensor: + r"""Pad a list of 1D tensor list to the same length. + + Args: + inputs (List[torch.Tensor]): List of 1D numpy arrays to pad. + pad_value (float): Value to use for padding. Default is 0.0. + + Returns: + torch.Tensor: Padded 2D numpy array of shape (len(inputs), max_len), where max_len is the length of the longest input array. + """ + max_len = max(x.size(0) for x in inputs) + padded_inputs = [nn.functional.pad(x, (0, max_len - x.size(0)), value=pad_value) for x in inputs] + return torch.stack(padded_inputs) + + +def pad_2D( + inputs: List[Tensor], maxlen: Union[int, None] = None, pad_value: float = 0.0, +) -> Tensor: + r"""Pad a list of 2D tensor arrays to the same length. + + Args: + inputs (List[torch.Tensor]): List of 2D numpy arrays to pad. + maxlen (Union[int, None]): Maximum length to pad the arrays to. If None, pad to the length of the longest array. Default is None. + pad_value (float): Value to use for padding. Default is 0.0. + + Returns: + torch.Tensor: Padded 3D numpy array of shape (len(inputs), max_len, input_dim), where max_len is the maximum length of the input arrays, and input_dim is the dimension of the input arrays. + """ + max_len = max(x.size(1) for x in inputs) if maxlen is None else maxlen + + padded_inputs = [nn.functional.pad(x, (0, max_len - x.size(1), 0, 0), value=pad_value) for x in inputs] + return torch.stack(padded_inputs) + + +def pad_3D(inputs: Union[Tensor, List[Tensor]], B: int, T: int, L: int) -> Tensor: + r"""Pad a 3D torch tensor to a specified shape. + + Args: + inputs (torch.Tensor): 3D numpy array to pad. + B (int): Batch size to pad the array to. + T (int): Time steps to pad the array to. + L (int): Length to pad the array to. + + Returns: + torch.Tensor: Padded 3D numpy array of shape (B, T, L), where B is the batch size, T is the time steps, and L is the length. + """ + if isinstance(inputs, list): + inputs_padded = torch.zeros(B, T, L, dtype=inputs[0].dtype) + for i, input_ in enumerate(inputs): + inputs_padded[i, :input_.size(0), :input_.size(1)] = input_ + + elif isinstance(inputs, torch.Tensor): + inputs_padded = torch.zeros(B, T, L, dtype=inputs.dtype) + inputs_padded[:inputs.size(0), :inputs.size(1), :inputs.size(2)] = inputs + + return inputs_padded diff --git a/vocoder_pretrained.pt b/vocoder_pretrained.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fc2226ce1b81e233bac845b25d126091255c1e8 --- /dev/null +++ b/vocoder_pretrained.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:390a532d0f2543cecccb2fcb48c383d75594d7a3802f748a2ce11c15ed4fd324 +size 391364041