UI improvement

pyproject.toml

@@ -12,14 +12,14 @@ dependencies = [
     "langfuse>=2.59.3",
     "litellm>=1.61.16",
     "loguru>=0.7.3",
-    "markitdown>=0.0.1a4",
+    "markitdown>=0.0.2",
     "matplotlib>=3.10.0",
     "openai>=1.63.0",
     "python-dotenv>=1.0.1",
     "torch>=2.6.0",
     "tqdm>=4.67.1",
     "transformers>=4.48.3",
-    "ruamel-yaml>=0.18.10",
+    "black>=25.1.0",
 ]
 
 [build-system]

uv.lock

@@ -117,6 +117,50 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b1/ae/4d289407515223677e7d105ec88bc707a115cce79464c6b56313beb0dd36/authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11", size = 231358 },
 ]
 
+[[package]]
+name = "azure-ai-documentintelligence"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "azure-core" },
+    { name = "isodate" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ca/fd/cd0d493e9dc93a5ce097db7508f1b2467a73dcc7022c235b409ce48b9679/azure_ai_documentintelligence-1.0.0.tar.gz", hash = "sha256:c8b6efc0fc7e65d7892c9585cfd256f7d8b3f2b46cecf92c75ab82e629eac253", size = 169420 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/84/a8/c9c66d4d04b8aee06ebdc9a6077736b222b9b2fe92364fed6f9a1c08ece0/azure_ai_documentintelligence-1.0.0-py3-none-any.whl", hash = "sha256:cdedb1a67c075f58f47a413ec5846bf8d532a83a71f0c51ec49ce9b5bfe2a519", size = 105454 },
+]
+
+[[package]]
+name = "azure-core"
+version = "1.32.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "requests" },
+    { name = "six" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cc/ee/668328306a9e963a5ad9f152cd98c7adad86c822729fd1d2a01613ad1e67/azure_core-1.32.0.tar.gz", hash = "sha256:22b3c35d6b2dae14990f6c1be2912bf23ffe50b220e708a28ab1bb92b1c730e5", size = 279128 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/39/83/325bf5e02504dbd8b4faa98197a44cdf8a325ef259b48326a2b6f17f8383/azure_core-1.32.0-py3-none-any.whl", hash = "sha256:eac191a0efb23bfa83fddf321b27b122b4ec847befa3091fa736a5c32c50d7b4", size = 198855 },
+]
+
+[[package]]
+name = "azure-identity"
+version = "1.20.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "azure-core" },
+    { name = "cryptography" },
+    { name = "msal" },
+    { name = "msal-extensions" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ee/89/7d170fab0b85d9650cdb7abda087e849644beb52bd28f6804620dd0cecd9/azure_identity-1.20.0.tar.gz", hash = "sha256:40597210d56c83e15031b0fe2ea3b26420189e1e7f3e20bdbb292315da1ba014", size = 264447 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/de/aa/819513c1dbef990af690bb5eefb5e337f8698d75dfdb7302528f50ce1994/azure_identity-1.20.0-py3-none-any.whl", hash = "sha256:5f23fc4889a66330e840bd78830287e14f3761820fe3c5f77ac875edcb9ec998", size = 188243 },
+]
+
 [[package]]
 name = "backoff"
 version = "2.2.1"
@@ -139,6 +183,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f9/49/6abb616eb3cbab6a7cca303dc02fdf3836de2e0b834bf966a7f5271a34d8/beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16", size = 186015 },
 ]
 
+[[package]]
+name = "black"
+version = "25.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "mypy-extensions" },
+    { name = "packaging" },
+    { name = "pathspec" },
+    { name = "platformdirs" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/83/71/3fe4741df7adf015ad8dfa082dd36c94ca86bb21f25608eb247b4afb15b2/black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b", size = 1650988 },
+    { url = "https://files.pythonhosted.org/packages/13/f3/89aac8a83d73937ccd39bbe8fc6ac8860c11cfa0af5b1c96d081facac844/black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc", size = 1453985 },
+    { url = "https://files.pythonhosted.org/packages/6f/22/b99efca33f1f3a1d2552c714b1e1b5ae92efac6c43e790ad539a163d1754/black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f", size = 1783816 },
+    { url = "https://files.pythonhosted.org/packages/18/7e/a27c3ad3822b6f2e0e00d63d58ff6299a99a5b3aee69fa77cd4b0076b261/black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba", size = 1440860 },
+    { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646 },
+]
+
 [[package]]
 name = "certifi"
 version = "2025.1.31"
@@ -204,6 +268,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 },
 ]
 
+[[package]]
+name = "cobble"
+version = "0.1.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984 },
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -303,6 +376,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4c/37/22ef7675bef4ffe9577b937ddca2e22791534cbbe11c30714972a91532dc/datasets-3.3.2-py3-none-any.whl", hash = "sha256:fdaf3d5d70242621210b044e9b9b15a56e908bfc3e9d077bcf5605ac390f70bd", size = 485360 },
 ]
 
+[[package]]
+name = "defusedxml"
+version = "0.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 },
+]
+
 [[package]]
 name = "dill"
 version = "0.3.8"
@@ -321,6 +403,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277 },
 ]
 
+[[package]]
+name = "et-xmlfile"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 },
+]
+
 [[package]]
 name = "fastapi"
 version = "0.115.11"
@@ -575,6 +666,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e", size = 26971 },
 ]
 
+[[package]]
+name = "isodate"
+version = "0.7.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320 },
+]
+
 [[package]]
 name = "itsdangerous"
 version = "2.2.0"
@@ -720,6 +820,43 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595 },
 ]
 
+[[package]]
+name = "lxml"
+version = "5.3.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ef/f6/c15ca8e5646e937c148e147244817672cf920b56ac0bf2cc1512ae674be8/lxml-5.3.1.tar.gz", hash = "sha256:106b7b5d2977b339f1e97efe2778e2ab20e99994cbb0ec5e55771ed0795920c8", size = 3678591 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3b/f4/5121aa9ee8e09b8b8a28cf3709552efe3d206ca51a20d6fa471b60bb3447/lxml-5.3.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e69add9b6b7b08c60d7ff0152c7c9a6c45b4a71a919be5abde6f98f1ea16421c", size = 8191889 },
+    { url = "https://files.pythonhosted.org/packages/0a/ca/8e9aa01edddc74878f4aea85aa9ab64372f46aa804d1c36dda861bf9eabf/lxml-5.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4e52e1b148867b01c05e21837586ee307a01e793b94072d7c7b91d2c2da02ffe", size = 4450685 },
+    { url = "https://files.pythonhosted.org/packages/b2/b3/ea40a5c98619fbd7e9349df7007994506d396b97620ced34e4e5053d3734/lxml-5.3.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4b382e0e636ed54cd278791d93fe2c4f370772743f02bcbe431a160089025c9", size = 5051722 },
+    { url = "https://files.pythonhosted.org/packages/3a/5e/375418be35f8a695cadfe7e7412f16520e62e24952ed93c64c9554755464/lxml-5.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2e49dc23a10a1296b04ca9db200c44d3eb32c8d8ec532e8c1fd24792276522a", size = 4786661 },
+    { url = "https://files.pythonhosted.org/packages/79/7c/d258eaaa9560f6664f9b426a5165103015bee6512d8931e17342278bad0a/lxml-5.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4399b4226c4785575fb20998dc571bc48125dc92c367ce2602d0d70e0c455eb0", size = 5311766 },
+    { url = "https://files.pythonhosted.org/packages/03/bc/a041415be4135a1b3fdf017a5d873244cc16689456166fbdec4b27fba153/lxml-5.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5412500e0dc5481b1ee9cf6b38bb3b473f6e411eb62b83dc9b62699c3b7b79f7", size = 4836014 },
+    { url = "https://files.pythonhosted.org/packages/32/88/047f24967d5e3fc97848ea2c207eeef0f16239cdc47368c8b95a8dc93a33/lxml-5.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c93ed3c998ea8472be98fb55aed65b5198740bfceaec07b2eba551e55b7b9ae", size = 4961064 },
+    { url = "https://files.pythonhosted.org/packages/3d/b5/ecf5a20937ecd21af02c5374020f4e3a3538e10a32379a7553fca3d77094/lxml-5.3.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:63d57fc94eb0bbb4735e45517afc21ef262991d8758a8f2f05dd6e4174944519", size = 4778341 },
+    { url = "https://files.pythonhosted.org/packages/a4/05/56c359e07275911ed5f35ab1d63c8cd3360d395fb91e43927a2ae90b0322/lxml-5.3.1-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:b450d7cabcd49aa7ab46a3c6aa3ac7e1593600a1a0605ba536ec0f1b99a04322", size = 5345450 },
+    { url = "https://files.pythonhosted.org/packages/b7/f4/f95e3ae12e9f32fbcde00f9affa6b0df07f495117f62dbb796a9a31c84d6/lxml-5.3.1-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:4df0ec814b50275ad6a99bc82a38b59f90e10e47714ac9871e1b223895825468", size = 4908336 },
+    { url = "https://files.pythonhosted.org/packages/c5/f8/309546aec092434166a6e11c7dcecb5c2d0a787c18c072d61e18da9eba57/lxml-5.3.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d184f85ad2bb1f261eac55cddfcf62a70dee89982c978e92b9a74a1bfef2e367", size = 4986049 },
+    { url = "https://files.pythonhosted.org/packages/71/1c/b951817cb5058ca7c332d012dfe8bc59dabd0f0a8911ddd7b7ea8e41cfbd/lxml-5.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b725e70d15906d24615201e650d5b0388b08a5187a55f119f25874d0103f90dd", size = 4860351 },
+    { url = "https://files.pythonhosted.org/packages/31/23/45feba8dae1d35fcca1e51b051f59dc4223cbd23e071a31e25f3f73938a8/lxml-5.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a31fa7536ec1fb7155a0cd3a4e3d956c835ad0a43e3610ca32384d01f079ea1c", size = 5421580 },
+    { url = "https://files.pythonhosted.org/packages/61/69/be245d7b2dbef81c542af59c97fcd641fbf45accf2dc1c325bae7d0d014c/lxml-5.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3c3c8b55c7fc7b7e8877b9366568cc73d68b82da7fe33d8b98527b73857a225f", size = 5285778 },
+    { url = "https://files.pythonhosted.org/packages/69/06/128af2ed04bac99b8f83becfb74c480f1aa18407b5c329fad457e08a1bf4/lxml-5.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d61ec60945d694df806a9aec88e8f29a27293c6e424f8ff91c80416e3c617645", size = 5054455 },
+    { url = "https://files.pythonhosted.org/packages/8a/2d/f03a21cf6cc75cdd083563e509c7b6b159d761115c4142abb5481094ed8c/lxml-5.3.1-cp312-cp312-win32.whl", hash = "sha256:f4eac0584cdc3285ef2e74eee1513a6001681fd9753b259e8159421ed28a72e5", size = 3486315 },
+    { url = "https://files.pythonhosted.org/packages/2b/9c/8abe21585d20ef70ad9cec7562da4332b764ed69ec29b7389d23dfabcea0/lxml-5.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:29bfc8d3d88e56ea0a27e7c4897b642706840247f59f4377d81be8f32aa0cfbf", size = 3816925 },
+]
+
+[[package]]
+name = "mammoth"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cobble" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d6/a6/27a13ba068cf3ff764d631b8dd71dee1b33040aa8c143f66ce902b7d1da0/mammoth-1.9.0.tar.gz", hash = "sha256:74f5dae10ca240fd9b7a0e1a6deaebe0aad23bc590633ef6f5e868aa9b7042a6", size = 50906 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d0/ab/f8e63fcabc127c6efd68b03633c189ee799a5304fa96c036a325a2894bcb/mammoth-1.9.0-py2.py3-none-any.whl", hash = "sha256:0eea277316586f0ca65d86834aec4de5a0572c83ec54b4991f9bb520a891150f", size = 52901 },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -747,19 +884,33 @@ wheels = [
 
 [[package]]
 name = "markitdown"
-version = "0.1.0a1"
+version = "0.0.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "azure-ai-documentintelligence" },
+    { name = "azure-identity" },
     { name = "beautifulsoup4" },
     { name = "charset-normalizer" },
+    { name = "mammoth" },
     { name = "markdownify" },
+    { name = "numpy" },
+    { name = "olefile" },
+    { name = "openai" },
+    { name = "openpyxl" },
+    { name = "pandas" },
     { name = "pathvalidate" },
+    { name = "pdfminer-six" },
     { name = "puremagic" },
+    { name = "pydub" },
+    { name = "python-pptx" },
     { name = "requests" },
+    { name = "speechrecognition" },
+    { name = "xlrd" },
+    { name = "youtube-transcript-api" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e4/20/f4d26a9946c851894ac099f690d55389c842e93a59951289654b4e8dec39/markitdown-0.1.0a1.tar.gz", hash = "sha256:f4d5b1eb4ee7757ec8e41c1205d6a33ae846ae366d0aec2ba56718b64b395b14", size = 28243 }
+sdist = { url = "https://files.pythonhosted.org/packages/69/bf/8d5ee5ceef8dc175b4c8714e6f5170e3bc38862eeb6a45f29433c64611c1/markitdown-0.0.2.tar.gz", hash = "sha256:678fc2c9bc4eded0941c01155251982cab1f86c8148c8ac7ca3869116d3d49d7", size = 21008 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1a/c5/1a37e25089c93ff701d226a9682e2270c83dd02c2547abd4078051114776/markitdown-0.1.0a1-py3-none-any.whl", hash = "sha256:a5b35730c8c9d7c46a91f43b5dcf82d1677c8b3d5b8accc3bd18209c139ae361", size = 43950 },
+    { url = "https://files.pythonhosted.org/packages/4a/f9/d0d6c6a11a16e2e22faf79bb4a86764110a8b391a510d6bd5cbb4ec57f59/markitdown-0.0.2-py3-none-any.whl", hash = "sha256:6ee5a6b70fbc36ba1586a40d5ab1ea8ea7f89bf37ebbaaa11013e026dc55b16e", size = 21278 },
 ]
 
 [[package]]
@@ -823,6 +974,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198 },
 ]
 
+[[package]]
+name = "msal"
+version = "1.31.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cryptography" },
+    { name = "pyjwt", extra = ["crypto"] },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/f3/cdf2681e83a73c3355883c2884b6ff2f2d2aadfc399c28e9ac4edc3994fd/msal-1.31.1.tar.gz", hash = "sha256:11b5e6a3f802ffd3a72107203e20c4eac6ef53401961b880af2835b723d80578", size = 145362 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/30/7c/489cd931a752d05753d730e848039f08f65f86237cf1b8724d0a1cbd700b/msal-1.31.1-py3-none-any.whl", hash = "sha256:29d9882de247e96db01386496d59f29035e5e841bcac892e6d7bf4390bf6bd17", size = 113216 },
+]
+
+[[package]]
+name = "msal-extensions"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "msal" },
+    { name = "portalocker" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2d/38/ad49272d0a5af95f7a0cb64a79bbd75c9c187f3b789385a143d8d537a5eb/msal_extensions-1.2.0.tar.gz", hash = "sha256:6f41b320bfd2933d631a215c91ca0dd3e67d84bd1a2f50ce917d5874ec646bef", size = 22391 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2c/69/314d887a01599669fb330da14e5c6ff5f138609e322812a942a74ef9b765/msal_extensions-1.2.0-py3-none-any.whl", hash = "sha256:cf5ba83a2113fa6dc011a254a72f1c223c88d7dfad74cc30617c4679a417704d", size = 19254 },
+]
+
 [[package]]
 name = "multidict"
 version = "6.1.0"
@@ -863,6 +1041,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351 },
 ]
 
+[[package]]
+name = "mypy-extensions"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 },
+]
+
 [[package]]
 name = "networkx"
 version = "3.4.2"
@@ -1008,6 +1195,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/87/20/199b8713428322a2f22b722c62b8cc278cc53dffa9705d744484b5035ee9/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a", size = 99144 },
 ]
 
+[[package]]
+name = "olefile"
+version = "0.47"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/69/1b/077b508e3e500e1629d366249c3ccb32f95e50258b231705c09e3c7a4366/olefile-0.47.zip", hash = "sha256:599383381a0bf3dfbd932ca0ca6515acd174ed48870cbf7fee123d698c192c1c", size = 112240 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/17/d3/b64c356a907242d719fc668b71befd73324e47ab46c8ebbbede252c154b2/olefile-0.47-py2.py3-none-any.whl", hash = "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f", size = 114565 },
+]
+
 [[package]]
 name = "openai"
 version = "1.65.5"
@@ -1027,6 +1223,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fc/8f/a178d73277bf2d838617fa20ba4ae6952e26074664aacb53ae4532a69588/openai-1.65.5-py3-none-any.whl", hash = "sha256:5948a504e7b4003d921cfab81273813793a31c25b1d7b605797c01757e0141f1", size = 474468 },
 ]
 
+[[package]]
+name = "openpyxl"
+version = "3.1.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "et-xmlfile" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 },
+]
+
 [[package]]
 name = "orjson"
 version = "3.10.15"
@@ -1078,6 +1286,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/29/d4/1244ab8edf173a10fd601f7e13b9566c1b525c4f365d6bee918e68381889/pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", size = 11504248 },
 ]
 
+[[package]]
+name = "pathspec"
+version = "0.12.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 },
+]
+
 [[package]]
 name = "pathvalidate"
 version = "3.2.3"
@@ -1087,6 +1304,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/50/14/c5a0e1a947909810fc4c043b84cac472b70e438148d34f5393be1bac663f/pathvalidate-3.2.3-py3-none-any.whl", hash = "sha256:5eaf0562e345d4b6d0c0239d0f690c3bd84d2a9a3c4c73b99ea667401b27bee1", size = 24130 },
 ]
 
+[[package]]
+name = "pdfminer-six"
+version = "20240706"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "charset-normalizer" },
+    { name = "cryptography" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e3/37/63cb918ffa21412dd5d54e32e190e69bfc340f3d6aa072ad740bec9386bb/pdfminer.six-20240706.tar.gz", hash = "sha256:c631a46d5da957a9ffe4460c5dce21e8431dabb615fee5f9f4400603a58d95a6", size = 7363505 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/67/7d/44d6b90e5a293d3a975cefdc4e12a932ebba814995b2a07e37e599dd27c6/pdfminer.six-20240706-py3-none-any.whl", hash = "sha256:f4f70e74174b4b3542fcb8406a210b6e2e27cd0f0b5fd04534a8cc0d8951e38c", size = 5615414 },
+]
+
 [[package]]
 name = "pillow"
 version = "11.1.0"
@@ -1106,6 +1336,27 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/37/f3/9b18362206b244167c958984b57c7f70a0289bfb59a530dd8af5f699b910/pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5", size = 2375240 },
 ]
 
+[[package]]
+name = "platformdirs"
+version = "4.3.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/13/fc/128cc9cb8f03208bdbf93d3aa862e16d376844a14f9a0ce5cf4507372de4/platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907", size = 21302 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb", size = 18439 },
+]
+
+[[package]]
+name = "portalocker"
+version = "2.10.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pywin32", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/fb/a70a4214956182e0d7a9099ab17d50bfcba1056188e9b14f35b9e2b62a0d/portalocker-2.10.1-py3-none-any.whl", hash = "sha256:53a5984ebc86a025552264b459b46a2086e269b21823cb572f8f28ee759e45bf", size = 18423 },
+]
+
 [[package]]
 name = "propcache"
 version = "0.3.0"
@@ -1221,6 +1472,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 },
 ]
 
+[[package]]
+name = "pyjwt"
+version = "2.10.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997 },
+]
+
+[package.optional-dependencies]
+crypto = [
+    { name = "cryptography" },
+]
+
 [[package]]
 name = "pyparsing"
 version = "3.2.1"
@@ -1260,6 +1525,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 },
 ]
 
+[[package]]
+name = "python-pptx"
+version = "1.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+    { name = "pillow" },
+    { name = "typing-extensions" },
+    { name = "xlsxwriter" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788 },
+]
+
 [[package]]
 name = "pytz"
 version = "2025.1"
@@ -1269,6 +1549,16 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 },
 ]
 
+[[package]]
+name = "pywin32"
+version = "309"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/20/2c/b0240b14ff3dba7a8a7122dc9bbf7fbd21ed0e8b57c109633675b5d1761f/pywin32-309-cp312-cp312-win32.whl", hash = "sha256:de9acacced5fa82f557298b1fed5fef7bd49beee04190f68e1e4783fbdc19926", size = 8790648 },
+    { url = "https://files.pythonhosted.org/packages/dd/11/c36884c732e2b3397deee808b5dac1abbb170ec37f94c6606fcb04d1e9d7/pywin32-309-cp312-cp312-win_amd64.whl", hash = "sha256:6ff9eebb77ffc3d59812c68db33c0a7817e1337e3537859499bd27586330fc9e", size = 9497399 },
+    { url = "https://files.pythonhosted.org/packages/18/9f/79703972958f8ba3fd38bc9bf1165810bd75124982419b0cc433a2894d46/pywin32-309-cp312-cp312-win_arm64.whl", hash = "sha256:619f3e0a327b5418d833f44dc87859523635cf339f86071cc65a13c07be3110f", size = 8454122 },
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.2"
@@ -1372,35 +1662,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a3/4f/0fce63e0f5cdd658e71e21abd17ac1bc9312741ebb8b3f74eeed2ebdf771/rpds_py-0.23.1-cp312-cp312-win_amd64.whl", hash = "sha256:b03a8d50b137ee758e4c73638b10747b7c39988eb8e6cd11abb7084266455165", size = 237426 },
 ]
 
-[[package]]
-name = "ruamel-yaml"
-version = "0.18.10"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "ruamel-yaml-clib", marker = "platform_python_implementation == 'CPython'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ea/46/f44d8be06b85bc7c4d8c95d658be2b68f27711f279bf9dd0612a5e4794f5/ruamel.yaml-0.18.10.tar.gz", hash = "sha256:20c86ab29ac2153f80a428e1254a8adf686d3383df04490514ca3b79a362db58", size = 143447 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c2/36/dfc1ebc0081e6d39924a2cc53654497f967a084a436bb64402dfce4254d9/ruamel.yaml-0.18.10-py3-none-any.whl", hash = "sha256:30f22513ab2301b3d2b577adc121c6471f28734d3d9728581245f1e76468b4f1", size = 117729 },
-]
-
-[[package]]
-name = "ruamel-yaml-clib"
-version = "0.2.12"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/20/84/80203abff8ea4993a87d823a5f632e4d92831ef75d404c9fc78d0176d2b5/ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f", size = 225315 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/48/41/e7a405afbdc26af961678474a55373e1b323605a4f5e2ddd4a80ea80f628/ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632", size = 133433 },
-    { url = "https://files.pythonhosted.org/packages/ec/b0/b850385604334c2ce90e3ee1013bd911aedf058a934905863a6ea95e9eb4/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d", size = 647362 },
-    { url = "https://files.pythonhosted.org/packages/44/d0/3f68a86e006448fb6c005aee66565b9eb89014a70c491d70c08de597f8e4/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c", size = 754118 },
-    { url = "https://files.pythonhosted.org/packages/52/a9/d39f3c5ada0a3bb2870d7db41901125dbe2434fa4f12ca8c5b83a42d7c53/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd", size = 706497 },
-    { url = "https://files.pythonhosted.org/packages/b0/fa/097e38135dadd9ac25aecf2a54be17ddf6e4c23e43d538492a90ab3d71c6/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31", size = 698042 },
-    { url = "https://files.pythonhosted.org/packages/ec/d5/a659ca6f503b9379b930f13bc6b130c9f176469b73b9834296822a83a132/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680", size = 745831 },
-    { url = "https://files.pythonhosted.org/packages/db/5d/36619b61ffa2429eeaefaab4f3374666adf36ad8ac6330d855848d7d36fd/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d", size = 715692 },
-    { url = "https://files.pythonhosted.org/packages/b1/82/85cb92f15a4231c89b95dfe08b09eb6adca929ef7df7e17ab59902b6f589/ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5", size = 98777 },
-    { url = "https://files.pythonhosted.org/packages/d7/8f/c3654f6f1ddb75daf3922c3d8fc6005b1ab56671ad56ffb874d908bfa668/ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4", size = 115523 },
-]
-
 [[package]]
 name = "ruff"
 version = "0.9.10"
@@ -1514,6 +1775,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9", size = 36186 },
 ]
 
+[[package]]
+name = "speechrecognition"
+version = "3.14.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e2/da/05607641a8db8fcc6898016fde7ea9b2e42d87cd1a1a275f0505a13389d8/speechrecognition-3.14.1.tar.gz", hash = "sha256:c767f8558e111a65e9a56905b04eaec2331f87d5011379381621f47aded6c4fe", size = 32858706 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/09/47/5dcfcd8a2c8c2981986fc196e98fc57bc1ecb5233b2d54dac0c0d448b019/SpeechRecognition-3.14.1-py3-none-any.whl", hash = "sha256:2b5d16a7dce2dbf5f90d9c4d5aefe96325518abdc963059ec16dad9e4f2c09d3", size = 32853180 },
+]
+
 [[package]]
 name = "starlette"
 version = "0.46.1"
@@ -1769,6 +2042,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594 },
 ]
 
+[[package]]
+name = "xlrd"
+version = "2.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/b3/19a2540d21dea5f908304375bd43f5ed7a4c28a370dc9122c565423e6b44/xlrd-2.0.1.tar.gz", hash = "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88", size = 100259 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a6/0c/c2a72d51fe56e08a08acc85d13013558a2d793028ae7385448a6ccdfae64/xlrd-2.0.1-py2.py3-none-any.whl", hash = "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd", size = 96531 },
+]
+
+[[package]]
+name = "xlsxwriter"
+version = "3.2.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/08/26f69d1e9264e8107253018de9fc6b96f9219817d01c5f021e927384a8d1/xlsxwriter-3.2.2.tar.gz", hash = "sha256:befc7f92578a85fed261639fb6cde1fd51b79c5e854040847dde59d4317077dc", size = 205202 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/07/df054f7413bdfff5e98f75056e4ed0977d0c8716424011fac2587864d1d3/XlsxWriter-3.2.2-py3-none-any.whl", hash = "sha256:272ce861e7fa5e82a4a6ebc24511f2cb952fde3461f6c6e1a1e81d3272db1471", size = 165121 },
+]
+
 [[package]]
 name = "xxhash"
 version = "3.5.0"
@@ -1825,7 +2116,7 @@ wheels = [
 [[package]]
 name = "yourbench"
 version = "0.2.0"
-source = { git = "https://github.com/huggingface/yourbench.git?rev=v0.2-alpha-space#58b5455c1f29b00015b306729dcc8affe3b0ffa6" }
+source = { git = "https://github.com/huggingface/yourbench.git?rev=v0.2-alpha-space#314985cd3ebf06193bd1bd4cbae34e7947f5667b" }
 dependencies = [
     { name = "asyncio" },
     { name = "datasets" },
@@ -1849,6 +2140,7 @@ version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "asyncio" },
+    { name = "black" },
     { name = "datasets" },
     { name = "gradio", extra = ["oauth"] },
     { name = "hf-transfer" },
@@ -1859,7 +2151,6 @@ dependencies = [
     { name = "matplotlib" },
     { name = "openai" },
     { name = "python-dotenv" },
-    { name = "ruamel-yaml" },
     { name = "torch" },
     { name = "tqdm" },
     { name = "transformers" },
@@ -1869,23 +2160,36 @@ dependencies = [
 [package.metadata]
 requires-dist = [
     { name = "asyncio", specifier = ">=3.4.3" },
+    { name = "black", specifier = ">=25.1.0" },
     { name = "datasets", specifier = ">=3.3.0" },
     { name = "gradio", extras = ["oauth"], specifier = ">=5.20.0" },
     { name = "hf-transfer", specifier = ">=0.1.9" },
     { name = "langfuse", specifier = ">=2.59.3" },
     { name = "litellm", specifier = ">=1.61.16" },
     { name = "loguru", specifier = ">=0.7.3" },
-    { name = "markitdown", specifier = ">=0.0.1a4" },
+    { name = "markitdown", specifier = ">=0.0.2" },
     { name = "matplotlib", specifier = ">=3.10.0" },
     { name = "openai", specifier = ">=1.63.0" },
     { name = "python-dotenv", specifier = ">=1.0.1" },
-    { name = "ruamel-yaml", specifier = ">=0.18.10" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "tqdm", specifier = ">=4.67.1" },
     { name = "transformers", specifier = ">=4.48.3" },
     { name = "yourbench", git = "https://github.com/huggingface/yourbench.git?rev=v0.2-alpha-space" },
 ]
 
+[[package]]
+name = "youtube-transcript-api"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "defusedxml" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/55ff16f7198bdf5204fd7be3c49122e07092a3da47bf4e1560989a4c0255/youtube_transcript_api-0.6.3.tar.gz", hash = "sha256:4d1f6451ae508390a5279f98519efb45e091bf60d3cca5ea0bb122800ab6a011", size = 612052 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/80/d4/be6fd091d29ae49d93813e598769e7ab453419a4de640e1755bf20911cce/youtube_transcript_api-0.6.3-py3-none-any.whl", hash = "sha256:297a74c1863d9df88f6885229f33a7eda61493d73ecb13ec80e876b65423e9b4", size = 622293 },
+]
+
 [[package]]
 name = "zipp"
 version = "3.21.0"

yourbench_space/app.py

@@ -1,13 +1,19 @@
+import os
 import sys
-from huggingface_hub import HfApi, whoami
-import requests
-import yaml
 import gradio as gr
 from loguru import logger
-from yourbench.pipeline import run_pipeline
+from huggingface_hub import HfApi, whoami
 
 from yourbench_space.config import generate_base_config, save_config
-from yourbench_space.utils import CONFIG_PATH, UPLOAD_DIRECTORY, SubprocessManager, save_files
+from yourbench_space.utils import (
+    CONFIG_PATH,
+    UPLOAD_DIRECTORY,
+    BASE_API_URLS,
+    AVAILABLE_MODELS,
+    DEFAULT_MODEL,
+    SubprocessManager,
+    save_files,
+)
 
 UPLOAD_DIRECTORY.mkdir(parents=True, exist_ok=True)
 
@@ -17,18 +23,110 @@ logger.add(sys.stderr, level="INFO")
 command = ["uv", "run", "yourbench", f"--config={CONFIG_PATH}"]
 manager = SubprocessManager(command)
 
+
+def prepare_task(oauth_token: gr.OAuthToken | None, model_token: str):
+    new_env = os.environ.copy()
+    # Override env token, when running in gradio space
+    if oauth_token:
+        new_env["HF_TOKEN"] = oauth_token.token
+    new_env["MODEL_API_KEY"] = model_token
+    manager.start_process(custom_env=new_env)
+
+
 def update_hf_org_dropdown(oauth_token: gr.OAuthToken | None) -> str:
     if oauth_token is None:
-        print("Please deploy this on Spaces and log in to list organizations.")
+        print(
+            "Please, deploy this on Spaces and log in to view the list of available organizations"
+        )
         return list()
-    user_info = whoami(oauth_token.token) 
+    user_info = whoami(oauth_token.token)
 
     org_names = [org["name"] for org in user_info["orgs"]]
     user_name = user_info["name"]
     org_names.insert(0, user_name)
     return gr.Dropdown(org_names, value=user_name, label="Organization")
 
+
 config_output = gr.Code(label="Generated Config", language="yaml")
+model_name = gr.Dropdown(
+    label="Model Name",
+    value=DEFAULT_MODEL,
+    choices=AVAILABLE_MODELS,
+    allow_custom_value=True,
+)
+base_url = gr.Textbox(
+    label="Model API Base URL",
+    value=BASE_API_URLS["huggingface"],
+    info="Use a custom API base URL for Hugging Face Inference Endpoints",
+)
+
+
+def make_models(model_name=None):
+    if model_name is None:
+        model_name = DEFAULT_MODEL
+
+    ingestion_model = gr.Dropdown(
+        label="Model for ingestion",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+    summarization_model = gr.Dropdown(
+        label="Model for summarization",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+    single_shot_question_generation_model = gr.Dropdown(
+        label="Model for single shot question generation",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+    multi_hop_question_generation_model = gr.Dropdown(
+        label="Model for multi hop question generation",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+    answer_generation_model = gr.Dropdown(
+        label="Model for answer generation",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+    judge_answers_model = gr.Dropdown(
+        label="Model for answer judging",
+        choices=AVAILABLE_MODELS,
+        value=model_name,
+        interactive=False,
+        allow_custom_value=True,
+    )
+
+    return [
+        ingestion_model,
+        summarization_model,
+        single_shot_question_generation_model,
+        multi_hop_question_generation_model,
+        answer_generation_model,
+        judge_answers_model,
+    ]
+
+
+(
+    ingestion_model,
+    summarization_model,
+    single_shot_question_generation_model,
+    multi_hop_question_generation_model,
+    answer_generation_model,
+    judge_answers_model,
+) = make_models()
+
 
 with gr.Blocks() as app:
     gr.Markdown("## YourBench Configuration")
@@ -36,37 +134,110 @@ with gr.Blocks() as app:
         login_btn = gr.LoginButton()
 
     with gr.Tab("Configuration"):
-        model_name = gr.Textbox(label="Model Name")
+        with gr.Accordion("Hugging Face"):
+            hf_org_dropdown = gr.Dropdown(
+                list(),
+                label="Organization",
+                allow_custom_value=True,
+            )
+            app.load(update_hf_org_dropdown, inputs=None, outputs=hf_org_dropdown)
+
+            hf_dataset_prefix = gr.Textbox(
+                label="Dataset Prefix",
+                value="yourbench",
+                info="Prefix applied to all datasets",
+            )
+            private_dataset = gr.Checkbox(
+                label="Private Dataset",
+                value=True,
+                info="Create private datasets (recommended by default)",
+            )
+
+        with gr.Accordion("Model"):
+            model_name.render()
+            # TODO handle this better
+            model_name.change(
+                make_models,
+                inputs=[model_name],
+                outputs=[
+                    ingestion_model,
+                    summarization_model,
+                    single_shot_question_generation_model,
+                    multi_hop_question_generation_model,
+                    answer_generation_model,
+                    judge_answers_model,
+                ],
+            )
+
+            provider = gr.Radio(
+                ["huggingface", "openrouter", "openai"],
+                value="huggingface",
+                label="Inference Provider",
+            )
+
+            def set_base_url(provider):
+                return gr.Textbox(
+                    label="Model API Base URL", value=BASE_API_URLS.get(provider, "")
+                )
 
-        hf_org_dropdown = gr.Dropdown(list(), label="Organization", allow_custom_value=True)
-        app.load(update_hf_org_dropdown, inputs=None, outputs=hf_org_dropdown)
+            provider.change(fn=set_base_url, inputs=provider, outputs=base_url)
+            model_api_key = gr.Textbox(label="Model API Key", type="password")
+            base_url.render()
+            max_concurrent_requests = gr.Radio(
+                [8, 16, 32], value=16, label="Max Concurrent Requests"
+            )
 
-        provider = gr.Dropdown(["openrouter", "openai", "huggingface"], value="huggingface", label="Provider", allow_custom_value=True)
-        base_url = gr.Textbox(label="Base URL")
-        api_key = gr.Textbox(label="API Key")
-        max_concurrent_requests = gr.Dropdown([8, 16, 32], value=16, label="Max Concurrent Requests")
-        preview_button = gr.Button("Generate Config")
+        with gr.Accordion("Stages"):
+            ingestion_model.render()
+            summarization_model.render()
+            single_shot_question_generation_model.render()
+            multi_hop_question_generation_model.render()
+            answer_generation_model.render()
+            judge_answers_model.render()
+
+        preview_button = gr.Button("Generate New Config")
         preview_button.click(
             generate_base_config,
-            inputs=[hf_org_dropdown, model_name, provider, base_url, api_key, max_concurrent_requests],
-            outputs=config_output
+            inputs=[
+                hf_org_dropdown,
+                model_name,
+                provider,
+                base_url,
+                model_api_key,
+                max_concurrent_requests,
+                hf_dataset_prefix,
+                private_dataset,
+                ingestion_model,
+                summarization_model,
+                single_shot_question_generation_model,
+                multi_hop_question_generation_model,
+                answer_generation_model,
+                judge_answers_model,
+            ],
+            outputs=config_output,
         )
 
-        save_button = gr.Button("Save Config")
-        save_button.click(save_config, inputs=[config_output], outputs=[gr.Textbox(label="Save Status")])
-
     with gr.Tab("Raw Configuration"):
         config_output.render()
-        save_button = gr.Button("Save Config")
-        save_button.click(save_config, inputs=[config_output], outputs=[gr.Textbox(label="Save Status")])
+        config_output.change(
+            fn=save_config,
+            inputs=[config_output],
+            outputs=[gr.Textbox(label="Save Status")],
+        )
 
     with gr.Tab("Files"):
-        file_input = gr.File(label="Upload text files", file_count="multiple", file_types=[".txt", ".md", ".html"])
+        file_input = gr.File(
+            label="Upload text files",
+            file_count="multiple",
+            file_types=[".txt", ".md", ".html"],
+        )
         output = gr.Textbox(label="Log")
         file_input.upload(save_files, file_input, output)
 
     with gr.Tab("Run Generation"):
-        log_output = gr.Code(label="Log Output", language=None, lines=20, interactive=False)
+        log_output = gr.Code(
+            label="Log Output", language=None, lines=20, interactive=False
+        )
         log_timer = gr.Timer(0.05, active=True)
         log_timer.tick(manager.read_and_get_output, outputs=log_output)
 
@@ -77,8 +248,8 @@ with gr.Blocks() as app:
 
         with gr.Row():
             start_button = gr.Button("Start Task")
-            start_button.click(manager.start_process)
-        
+            start_button.click(prepare_task, inputs=[model_api_key])
+
             stop_button = gr.Button("Stop Task")
             stop_button.click(manager.stop_process)
 

yourbench_space/config.py

@@ -1,105 +1,126 @@
-import yaml
-import gradio as gr
 
+import yaml
 from yourbench_space.utils import CONFIG_PATH
 
 
-def generate_base_config(hf_org, model_name, provider, base_url, api_key, max_concurrent_requests):
+
+def generate_base_config(
+    hf_org,
+    model_name,
+    provider,
+    base_url,
+    model_api_key,
+    max_concurrent_requests,
+    hf_dataset_prefix,
+    private_dataset,
+    ingestion_model,
+    summarization_model,
+    single_shot_question_generation_model,
+    multi_hop_question_generation_model,
+    answer_generation_model,
+    judge_answers_model,
+):
     config = {
         "hf_configuration": {
             "token": "$HF_TOKEN",
-            "private": True,
-            "hf_organization": hf_org
+            "private": private_dataset,
+            "hf_organization": hf_org,
+        },
+        "model_list": [
+            {
+                "model_name": model_name,
+                "provider": provider,
+                "base_url": base_url,
+                "api_key": "$MODEL_API_KEY",
+                "max_concurrent_requests": max_concurrent_requests,
+            }
+        ],
+        "model_roles": {
+            role: [model_name]
+            for role in [
+                "ingestion",
+                "summarization",
+                "single_shot_question_generation",
+                "multi_hop_question_generation",
+                "answer_generation",
+                "judge_answers",
+            ]
         },
-        "model_list": [{
-            "model_name": model_name,
-            "provider": provider,
-            "base_url": base_url,
-            "api_key": api_key,
-            "max_concurrent_requests": max_concurrent_requests
-        }],
-        "model_roles": {role: [model_name] for role in [
-            "ingestion", "summarization", "single_shot_question_generation",
-            "multi_hop_question_generation", "answer_generation", "judge_answers"
-        ]},
         "inference_config": {"max_concurrent_requests": 16},
         "pipeline": {
             "ingestion": {
                 "source_documents_dir": "/app/uploaded_files",
                 "output_dir": "/app/ingested",
-                "run": True
+                "run": True,
             },
             "upload_ingest_to_hub": {
                 "source_documents_dir": "/app/ingested",
-                "hub_dataset_name": "test_ingested_documents",
-                "local_dataset_path": "/app/ingested_dataset",
-                "run": True
+                "hub_dataset_name": f"{hf_dataset_prefix}_ingested_documents",
+                "run": True,
             },
             "summarization": {
-                "source_dataset_name": "test_ingested_documents",
-                "output_dataset_name": "test_summaries",
-                "local_dataset_path": "/results/test_summaries",
+                "source_dataset_name": f"{hf_dataset_prefix}_ingested_documents",
+                "output_dataset_name": f"{hf_dataset_prefix}_summaries",
                 "concat_existing_dataset": False,
-                "run": True
+                "run": True,
             },
             "chunking": {
-                "source_dataset_name": "test_summaries",
-                "output_dataset_name": "test_chunked_documents",
-                "local_dataset_path": "/results/test_chunked_documents",
+                "source_dataset_name": f"{hf_dataset_prefix}_summaries",
+                "output_dataset_name": f"{hf_dataset_prefix}_chunked_documents",
                 "concat_existing_dataset": False,
                 "chunking_configuration": {
                     "l_min_tokens": 64,
                     "l_max_tokens": 128,
                     "tau_threshold": 0.3,
                     "h_min": 2,
-                    "h_max": 4
+                    "h_max": 4,
                 },
-                "run": True
+                "run": True,
             },
             "single_shot_question_generation": {
-                "source_dataset_name": "test_chunked_documents",
-                "output_dataset_name": "test_single_shot_questions",
-                "local_dataset_path": "/results/test_single_shot_questions",
+                "source_dataset_name": f"{hf_dataset_prefix}_chunked_documents",
+                "output_dataset_name": f"{hf_dataset_prefix}_single_shot_questions",
                 "diversification_seed": "24 year old adult",
                 "concat_existing_dataset": False,
-                "run": True
+                "run": True,
             },
             "multi_hop_question_generation": {
-                "source_dataset_name": "test_chunked_documents",
-                "output_dataset_name": "test_multi_hop_questions",
-                "local_dataset_path": "/results/test_multi_hop_questions",
+                "source_dataset_name": f"{hf_dataset_prefix}_chunked_documents",
+                "output_dataset_name": f"{hf_dataset_prefix}_multi_hop_questions",
                 "concat_existing_dataset": False,
-                "run": True
+                "run": True,
             },
             "answer_generation": {
-                "run": True,
-                "question_dataset_name": "test_single_shot_questions",
-                "output_dataset_name": "test_answered_questions",
-                "local_dataset_path": "/results/test_answered_questions",
+                "question_dataset_name": f"{hf_dataset_prefix}_single_shot_questions",
+                "output_dataset_name": f"{hf_dataset_prefix}_answered_questions",
                 "concat_existing_dataset": False,
-                "strategies": [{
-                    "name": "zeroshot",
-                    "prompt": "ZEROSHOT_QA_USER_PROMPT",
-                    "model_name": model_name
-                }, {
-                    "name": "gold",
-                    "prompt": "GOLD_QA_USER_PROMPT",
-                    "model_name": model_name
-                }]
+                "strategies": [
+                    {
+                        "name": "zeroshot",
+                        "prompt": "ZEROSHOT_QA_USER_PROMPT",
+                        "model_name": model_name,
+                    },
+                    {
+                        "name": "gold",
+                        "prompt": "GOLD_QA_USER_PROMPT",
+                        "model_name": model_name,
+                    },
+                ],
+                "run": True,
             },
             "judge_answers": {
-                "run": True,
-                "source_judge_dataset_name": "test_answered_questions",
-                "output_judged_dataset_name": "test_judged_comparisons",
-                "local_dataset_path": "/results/test_judged_comparisons",
+                "source_judge_dataset_name": f"{hf_dataset_prefix}_answered_questions",
+                "output_judged_dataset_name": f"{hf_dataset_prefix}_judged_comparisons",
                 "concat_existing_dataset": False,
                 "comparing_strategies": [["zeroshot", "gold"]],
                 "chunk_column_index": 0,
-                "random_seed": 42
-            }
-        }
+                "random_seed": 42,
+                "run": True,
+            },
+        },
     }
-    return yaml.dump(config, default_flow_style=False)
+    return yaml.dump(config, sort_keys=False)
+
 
 def save_config(yaml_text):
     with open(CONFIG_PATH, "w") as file:

yourbench_space/utils.py

@@ -2,13 +2,23 @@ import io
 import os
 import pathlib
 import shutil
-import gradio as gr
 from loguru import logger
 import subprocess
 
 UPLOAD_DIRECTORY = pathlib.Path("/app/uploaded_files")
 CONFIG_PATH = pathlib.Path("/app/yourbench_config.yml")
 
+AVAILABLE_MODELS = [
+    "mistralai/Mistral-Small-24B-Instruct-2501",
+    "meta-llama/Llama-3.3-70B-Instruct",
+]
+DEFAULT_MODEL = AVAILABLE_MODELS[0]
+
+BASE_API_URLS = {
+    "huggingface": "https://router.huggingface.co/hf-inference/v1",
+    "openrouter":  "https://openrouter.ai/api/v1",
+    "openai": "https://api.openai.com/v1/",
+}
 
 def save_files(files: list[str]):
     saved_paths = [shutil.move(str(pathlib.Path(file)), str(UPLOAD_DIRECTORY / pathlib.Path(file).name)) for file in files]
@@ -21,19 +31,13 @@ class SubprocessManager:
         self.process = None
         self.output_stream = io.StringIO()
 
-    def start_process(self, oauth_token: gr.OAuthToken | None):
+    def start_process(self, custom_env: dict | None):
         """Start the subprocess."""
         if self.is_running():
             logger.info("Process is already running")
             return
 
         self.output_stream = io.StringIO()
-
-        new_env = os.environ.copy()
-        # Override env token, when running in gradio space
-        if oauth_token:
-            new_env["HF_TOKEN"] = oauth_token.token
-
         self.process = subprocess.Popen(
             self.command,
             stdout=subprocess.PIPE,
@@ -41,7 +45,7 @@ class SubprocessManager:
             text=True,
             bufsize=1,
             start_new_session=True,
-            env=new_env
+            env=custom_env
         )
         os.set_blocking(self.process.stdout.fileno(), False)
         logger.info("Started the process")