Add files using upload-large-folder tool

.gitattributes

@@ -391,3 +391,4 @@ tuning-competition-baseline/.venv/lib/python3.11/site-packages/nvidia/cudnn/lib/
 .venv/lib/python3.11/site-packages/numpy/lib/__pycache__/function_base.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.11/site-packages/numpy/lib/__pycache__/npyio.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.11/site-packages/numpy/lib/tests/__pycache__/test_function_base.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+.venv/lib/python3.11/site-packages/mistral_common/data/mistral_instruct_tokenizer_240323.model.v3 filter=lfs diff=lfs merge=lfs -text

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/METADATA

@@ -0,0 +1,138 @@
+Metadata-Version: 2.3
+Name: cryptography
+Version: 43.0.3
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: BSD
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Security :: Cryptography
+Requires-Dist: cffi >=1.12 ; platform_python_implementation != 'PyPy'
+Requires-Dist: bcrypt >=3.1.5 ; extra == 'ssh'
+Requires-Dist: nox ; extra == 'nox'
+Requires-Dist: cryptography-vectors ==43.0.3 ; extra == 'test'
+Requires-Dist: pytest >=6.2.0 ; extra == 'test'
+Requires-Dist: pytest-benchmark ; extra == 'test'
+Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: pytest-xdist ; extra == 'test'
+Requires-Dist: pretend ; extra == 'test'
+Requires-Dist: certifi ; extra == 'test'
+Requires-Dist: pytest-randomly ; extra == 'test-randomorder'
+Requires-Dist: sphinx >=5.3.0 ; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme >=1.1.1 ; extra == 'docs'
+Requires-Dist: pyenchant >=1.6.11 ; extra == 'docstest'
+Requires-Dist: readme-renderer ; extra == 'docstest'
+Requires-Dist: sphinxcontrib-spelling >=4.0.1 ; extra == 'docstest'
+Requires-Dist: build ; extra == 'sdist'
+Requires-Dist: ruff ; extra == 'pep8test'
+Requires-Dist: mypy ; extra == 'pep8test'
+Requires-Dist: check-sdist ; extra == 'pep8test'
+Requires-Dist: click ; extra == 'pep8test'
+Provides-Extra: ssh
+Provides-Extra: nox
+Provides-Extra: test
+Provides-Extra: test-randomorder
+Provides-Extra: docs
+Provides-Extra: docstest
+Provides-Extra: sdist
+Provides-Extra: pep8test
+License-File: LICENSE
+License-File: LICENSE.APACHE
+License-File: LICENSE.BSD
+Summary: cryptography is a package which provides cryptographic recipes and primitives to Python developers.
+Author: The cryptography developers <[email protected]>
+Author-email: The Python Cryptographic Authority and individual contributors <[email protected]>
+License: Apache-2.0 OR BSD-3-Clause
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst; charset=UTF-8
+Project-URL: homepage, https://github.com/pyca/cryptography
+Project-URL: documentation, https://cryptography.io/
+Project-URL: source, https://github.com/pyca/cryptography/
+Project-URL: issues, https://github.com/pyca/cryptography/issues
+Project-URL: changelog, https://cryptography.io/en/latest/changelog/
+
+pyca/cryptography
+=================
+
+.. image:: https://img.shields.io/pypi/v/cryptography.svg
+    :target: https://pypi.org/project/cryptography/
+    :alt: Latest Version
+
+.. image:: https://readthedocs.org/projects/cryptography/badge/?version=latest
+    :target: https://cryptography.io
+    :alt: Latest Docs
+
+.. image:: https://github.com/pyca/cryptography/workflows/CI/badge.svg?branch=main
+    :target: https://github.com/pyca/cryptography/actions?query=workflow%3ACI+branch%3Amain
+
+
+``cryptography`` is a package which provides cryptographic recipes and
+primitives to Python developers. Our goal is for it to be your "cryptographic
+standard library". It supports Python 3.7+ and PyPy3 7.3.11+.
+
+``cryptography`` includes both high level recipes and low level interfaces to
+common cryptographic algorithms such as symmetric ciphers, message digests, and
+key derivation functions. For example, to encrypt something with
+``cryptography``'s high level symmetric encryption recipe:
+
+.. code-block:: pycon
+
+    >>> from cryptography.fernet import Fernet
+    >>> # Put this somewhere safe!
+    >>> key = Fernet.generate_key()
+    >>> f = Fernet(key)
+    >>> token = f.encrypt(b"A really secret message. Not for prying eyes.")
+    >>> token
+    b'...'
+    >>> f.decrypt(token)
+    b'A really secret message. Not for prying eyes.'
+
+You can find more information in the `documentation`_.
+
+You can install ``cryptography`` with:
+
+.. code-block:: console
+
+    $ pip install cryptography
+
+For full details see `the installation documentation`_.
+
+Discussion
+~~~~~~~~~~
+
+If you run into bugs, you can file them in our `issue tracker`_.
+
+We maintain a `cryptography-dev`_ mailing list for development discussion.
+
+You can also join ``#pyca`` on ``irc.libera.chat`` to ask questions or get
+involved.
+
+Security
+~~~~~~~~
+
+Need to report a security issue? Please consult our `security reporting`_
+documentation.
+
+
+.. _`documentation`: https://cryptography.io/
+.. _`the installation documentation`: https://cryptography.io/en/latest/installation/
+.. _`issue tracker`: https://github.com/pyca/cryptography/issues
+.. _`cryptography-dev`: https://mail.python.org/mailman/listinfo/cryptography-dev
+.. _`security reporting`: https://cryptography.io/en/latest/security/
+

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/RECORD

@@ -0,0 +1,173 @@
+cryptography-43.0.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+cryptography-43.0.3.dist-info/METADATA,sha256=6zbg5CUehHnvNpZEQHVe8ivt1BG6h6k_cm-o5bsOZLA,5440
+cryptography-43.0.3.dist-info/RECORD,,
+cryptography-43.0.3.dist-info/WHEEL,sha256=5SNCVD9cb88a-xAIrDHIo1CvpgNriOYcNgb4b8rPcOw,107
+cryptography-43.0.3.dist-info/license_files/LICENSE,sha256=Pgx8CRqUi4JTO6mP18u0BDLW8amsv4X1ki0vmak65rs,197
+cryptography-43.0.3.dist-info/license_files/LICENSE.APACHE,sha256=qsc7MUj20dcRHbyjIJn2jSbGRMaBOuHk8F9leaomY_4,11360
+cryptography-43.0.3.dist-info/license_files/LICENSE.BSD,sha256=YCxMdILeZHndLpeTzaJ15eY9dz2s0eymiSMqtwCPtPs,1532
+cryptography/__about__.py,sha256=-FkHKD9mSuEfH37wsSKnQzJZmL5zUAUTpB5OeUQjPE0,445
+cryptography/__init__.py,sha256=mthuUrTd4FROCpUYrTIqhjz6s6T9djAZrV7nZ1oMm2o,364
+cryptography/__pycache__/__about__.cpython-311.pyc,,
+cryptography/__pycache__/__init__.cpython-311.pyc,,
+cryptography/__pycache__/exceptions.cpython-311.pyc,,
+cryptography/__pycache__/fernet.cpython-311.pyc,,
+cryptography/__pycache__/utils.cpython-311.pyc,,
+cryptography/exceptions.py,sha256=835EWILc2fwxw-gyFMriciC2SqhViETB10LBSytnDIc,1087
+cryptography/fernet.py,sha256=aPj82w-Z_1GBXUtWRUsZdVbMwRo5Mbjj0wkA9wG4rkw,6696
+cryptography/hazmat/__init__.py,sha256=5IwrLWrVp0AjEr_4FdWG_V057NSJGY_W4egNNsuct0g,455
+cryptography/hazmat/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/__pycache__/_oid.cpython-311.pyc,,
+cryptography/hazmat/_oid.py,sha256=e9yLmxtdQtuL94ztQv3SGtt_ea1Mx6aUwGftJsP6EXk,15201
+cryptography/hazmat/backends/__init__.py,sha256=O5jvKFQdZnXhKeqJ-HtulaEL9Ni7mr1mDzZY5kHlYhI,361
+cryptography/hazmat/backends/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/backends/openssl/__init__.py,sha256=p3jmJfnCag9iE5sdMrN6VvVEu55u46xaS_IjoI0SrmA,305
+cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-311.pyc,,
+cryptography/hazmat/backends/openssl/backend.py,sha256=pUXUbugLwMm2Gls-h5U5fw2RvepaNjEvnao6CTmL1xQ,9648
+cryptography/hazmat/bindings/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180
+cryptography/hazmat/bindings/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/bindings/_rust.abi3.so,sha256=QrghdFa6x-vG5lFvGVfv-slqoy0UA9a8eHmtp9hzGCk,10862344
+cryptography/hazmat/bindings/_rust/__init__.pyi,sha256=wb1OT76lG19vjq97_q2MM3qdJlQhyloXfVbKFDmRse4,737
+cryptography/hazmat/bindings/_rust/_openssl.pyi,sha256=mpNJLuYLbCVrd5i33FBTmWwL_55Dw7JPkSLlSX9Q7oI,230
+cryptography/hazmat/bindings/_rust/asn1.pyi,sha256=BrGjC8J6nwuS-r3EVcdXJB8ndotfY9mbQYOfpbPG0HA,354
+cryptography/hazmat/bindings/_rust/exceptions.pyi,sha256=exXr2xw_0pB1kk93cYbM3MohbzoUkjOms1ZMUi0uQZE,640
+cryptography/hazmat/bindings/_rust/ocsp.pyi,sha256=R-xJ-XmJZ1lOk-fWHHvRnP3QNTCFnKv-l3xlNWfLVt4,868
+cryptography/hazmat/bindings/_rust/openssl/__init__.pyi,sha256=Lvn250QMdPyeF-hoBF6rkQgHLBJxVauXCb8i8uYTomQ,1368
+cryptography/hazmat/bindings/_rust/openssl/aead.pyi,sha256=i0gA3jUQ4rkJXTGGZrq-AuY-VQLN31lyDeWuDZ0zJYw,2553
+cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi,sha256=iK0ZhQ-WyCQbjaraaFgK6q4PpD-7Rf5RDHkFD3YEW_g,1301
+cryptography/hazmat/bindings/_rust/openssl/cmac.pyi,sha256=nPH0X57RYpsAkRowVpjQiHE566ThUTx7YXrsadmrmHk,564
+cryptography/hazmat/bindings/_rust/openssl/dh.pyi,sha256=Z3TC-G04-THtSdAOPLM1h2G7ml5bda1ElZUcn5wpuhk,1564
+cryptography/hazmat/bindings/_rust/openssl/dsa.pyi,sha256=qBtkgj2albt2qFcnZ9UDrhzoNhCVO7HTby5VSf1EXMI,1299
+cryptography/hazmat/bindings/_rust/openssl/ec.pyi,sha256=zJy0pRa5n-_p2dm45PxECB_-B6SVZyNKfjxFDpPqT38,1691
+cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi,sha256=OJsrblS2nHptZctva-pAKFL5q8yPEAkhmjPZpJ6TA94,493
+cryptography/hazmat/bindings/_rust/openssl/ed448.pyi,sha256=SkPHK2HdbYN02TVQEUOgW3iTdiEY7HBE4DijpdkAzmk,475
+cryptography/hazmat/bindings/_rust/openssl/hashes.pyi,sha256=J8HoN0GdtPcjRAfNHr5Elva_nkmQfq63L75_z9dd8Uc,573
+cryptography/hazmat/bindings/_rust/openssl/hmac.pyi,sha256=ZmLJ73pmxcZFC1XosWEiXMRYtvJJor3ZLdCQOJu85Cw,662
+cryptography/hazmat/bindings/_rust/openssl/kdf.pyi,sha256=wPS5c7NLspM2632II0I4iH1RSxZvSRtBOVqmpyQATfk,544
+cryptography/hazmat/bindings/_rust/openssl/keys.pyi,sha256=JSrlGNaW49ZCZ1hcb-YJdS1EAbsMwRbVEcLL0P9OApA,872
+cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi,sha256=9iogF7Q4i81IkOS-IMXp6HvxFF_3cNy_ucrAjVQnn14,540
+cryptography/hazmat/bindings/_rust/openssl/rsa.pyi,sha256=2OQCNSXkxgc-3uw1xiCCloIQTV6p9_kK79Yu0rhZgPc,1364
+cryptography/hazmat/bindings/_rust/openssl/x25519.pyi,sha256=2BKdbrddM_9SMUpdvHKGhb9MNjURCarPxccbUDzHeoA,484
+cryptography/hazmat/bindings/_rust/openssl/x448.pyi,sha256=AoRMWNvCJTiH5L-lkIkCdPlrPLUdJvvfXpIvf1GmxpM,466
+cryptography/hazmat/bindings/_rust/pkcs12.pyi,sha256=afhB_6M8xI1MIE5vxkaDF1jSxA48ib1--NiOxtf6boM,1394
+cryptography/hazmat/bindings/_rust/pkcs7.pyi,sha256=QCmuA0IgDr4iOecUOXgUUeh3BAjJx8ubjz__EnNbyGY,972
+cryptography/hazmat/bindings/_rust/test_support.pyi,sha256=Xo1Gd7bh9rU4HuIS4pm9UwCY6IS1gInvFwmhABLOVO4,936
+cryptography/hazmat/bindings/_rust/x509.pyi,sha256=WLrGmqmFss8dXKhlG_J9nVhoCcodR72xJdCoxEuBtjY,3551
+cryptography/hazmat/bindings/openssl/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180
+cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-311.pyc,,
+cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-311.pyc,,
+cryptography/hazmat/bindings/openssl/_conditional.py,sha256=dkGKGU-22uR2ZKeOOwaSxEJCGaafgUjb2romWcu03QE,5163
+cryptography/hazmat/bindings/openssl/binding.py,sha256=e1gnFAZBPrkJ3CsiZV-ug6kaPdNTAEROaUFiFrUh71M,4042
+cryptography/hazmat/decrepit/__init__.py,sha256=wHCbWfaefa-fk6THSw9th9fJUsStJo7245wfFBqmduA,216
+cryptography/hazmat/decrepit/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/decrepit/ciphers/__init__.py,sha256=wHCbWfaefa-fk6THSw9th9fJUsStJo7245wfFBqmduA,216
+cryptography/hazmat/decrepit/ciphers/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/decrepit/ciphers/__pycache__/algorithms.cpython-311.pyc,,
+cryptography/hazmat/decrepit/ciphers/algorithms.py,sha256=HWA4PKDS2w4D2dQoRerpLRU7Kntt5vJeJC7j--AlZVU,2520
+cryptography/hazmat/primitives/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180
+cryptography/hazmat/primitives/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/_asymmetric.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/_cipheralgorithm.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/_serialization.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/cmac.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/constant_time.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/hashes.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/hmac.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/keywrap.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/padding.cpython-311.pyc,,
+cryptography/hazmat/primitives/__pycache__/poly1305.cpython-311.pyc,,
+cryptography/hazmat/primitives/_asymmetric.py,sha256=RhgcouUB6HTiFDBrR1LxqkMjpUxIiNvQ1r_zJjRG6qQ,532
+cryptography/hazmat/primitives/_cipheralgorithm.py,sha256=gKa0WrLz6K4fqhnGbfBYKDSxgLxsPU0uj_EK2UT47W4,1495
+cryptography/hazmat/primitives/_serialization.py,sha256=qrozc8fw2WZSbjk3DAlSl3ResxpauwJ74ZgGoUL-mj0,5142
+cryptography/hazmat/primitives/asymmetric/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180
+cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/types.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-311.pyc,,
+cryptography/hazmat/primitives/asymmetric/dh.py,sha256=OOCjMClH1Bf14Sy7jAdwzEeCxFPb8XUe2qePbExvXwc,3420
+cryptography/hazmat/primitives/asymmetric/dsa.py,sha256=xBwdf0pZOgvqjUKcO7Q0L3NxwalYj0SJDUqThemhSmI,3945
+cryptography/hazmat/primitives/asymmetric/ec.py,sha256=lwZmtAwi3PM8lsY1MsNaby_bVi--49OCxwE_1yqKC-A,10428
+cryptography/hazmat/primitives/asymmetric/ed25519.py,sha256=kl63fg7myuMjNTmMoVFeH6iVr0x5FkjNmggxIRTloJk,3423
+cryptography/hazmat/primitives/asymmetric/ed448.py,sha256=2UzEDzzfkPn83UFVFlMZfIMbAixxY09WmQyrwinWTn8,3456
+cryptography/hazmat/primitives/asymmetric/padding.py,sha256=eZcvUqVLbe3u48SunLdeniaPlV4-k6pwBl67OW4jSy8,2885
+cryptography/hazmat/primitives/asymmetric/rsa.py,sha256=nW_Ko7PID9UBJF10GVJOc_1L00ymFsfZDUJYtM5kfGQ,7637
+cryptography/hazmat/primitives/asymmetric/types.py,sha256=LnsOJym-wmPUJ7Knu_7bCNU3kIiELCd6krOaW_JU08I,2996
+cryptography/hazmat/primitives/asymmetric/utils.py,sha256=DPTs6T4F-UhwzFQTh-1fSEpQzazH2jf2xpIro3ItF4o,790
+cryptography/hazmat/primitives/asymmetric/x25519.py,sha256=VGYuRdIYuVBtizpFdNWd2bTrT10JRa1admQdBr08xz8,3341
+cryptography/hazmat/primitives/asymmetric/x448.py,sha256=GKKJBqYLr03VewMF18bXIM941aaWcZIQ4rC02GLLEmw,3374
+cryptography/hazmat/primitives/ciphers/__init__.py,sha256=eyEXmjk6_CZXaOPYDr7vAYGXr29QvzgWL2-4CSolLFs,680
+cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-311.pyc,,
+cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-311.pyc,,
+cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-311.pyc,,
+cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-311.pyc,,
+cryptography/hazmat/primitives/ciphers/aead.py,sha256=Fzlyx7w8KYQakzDp1zWgJnIr62zgZrgVh1u2h4exB54,634
+cryptography/hazmat/primitives/ciphers/algorithms.py,sha256=QvBMDmphRZfNmykij58L5eDkd_2NnCzIpJpyX2QwMxc,4223
+cryptography/hazmat/primitives/ciphers/base.py,sha256=tg-XNaKUyETBi7ounGDEL1_ICn-s4FF9LR7moV58blI,4211
+cryptography/hazmat/primitives/ciphers/modes.py,sha256=BFpxEGSaxoeZjrQ4sqpyPDvKClrqfDKIBv7kYtFURhE,8192
+cryptography/hazmat/primitives/cmac.py,sha256=sz_s6H_cYnOvx-VNWdIKhRhe3Ymp8z8J0D3CBqOX3gg,338
+cryptography/hazmat/primitives/constant_time.py,sha256=xdunWT0nf8OvKdcqUhhlFKayGp4_PgVJRU2W1wLSr_A,422
+cryptography/hazmat/primitives/hashes.py,sha256=EvDIJBhj83Z7f-oHbsA0TzZLFSDV_Yv8hQRdM4o8FD0,5091
+cryptography/hazmat/primitives/hmac.py,sha256=RpB3z9z5skirCQrm7zQbtnp9pLMnAjrlTUvKqF5aDDc,423
+cryptography/hazmat/primitives/kdf/__init__.py,sha256=4XibZnrYq4hh5xBjWiIXzaYW6FKx8hPbVaa_cB9zS64,750
+cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-311.pyc,,
+cryptography/hazmat/primitives/kdf/concatkdf.py,sha256=bcn4NGXse-EsFl7nlU83e5ilop7TSHcX-CJJS107W80,3686
+cryptography/hazmat/primitives/kdf/hkdf.py,sha256=uhN5L87w4JvtAqQcPh_Ji2TPSc18IDThpaYJiHOWy3A,3015
+cryptography/hazmat/primitives/kdf/kbkdf.py,sha256=eSuLK1sATkamgCAit794jLr7sDNlu5X0USdcWhwJdmk,9146
+cryptography/hazmat/primitives/kdf/pbkdf2.py,sha256=Xj3YIeX30h2BUaoJAtOo1RMXV_em0-eCG0PU_0FHJzM,1950
+cryptography/hazmat/primitives/kdf/scrypt.py,sha256=4QONhjxA_ZtuQtQ7QV3FnbB8ftrFnM52B4HPfV7hFys,2354
+cryptography/hazmat/primitives/kdf/x963kdf.py,sha256=wCpWmwQjZ2vAu2rlk3R_PX0nINl8WGXYBmlyMOC5iPw,1992
+cryptography/hazmat/primitives/keywrap.py,sha256=XV4Pj2fqSeD-RqZVvY2cA3j5_7RwJSFygYuLfk2ujCo,5650
+cryptography/hazmat/primitives/padding.py,sha256=QUq0n-EAgEan9aQzuTsiJYGKbWiK1nSHkcYjDF1L1ok,5518
+cryptography/hazmat/primitives/poly1305.py,sha256=P5EPQV-RB_FJPahpg01u0Ts4S_PnAmsroxIGXbGeRRo,355
+cryptography/hazmat/primitives/serialization/__init__.py,sha256=jyNx_7NcOEbVRBY4nP9ks0IVXBafbcYnTK27vafPLW8,1653
+cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-311.pyc,,
+cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-311.pyc,,
+cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-311.pyc,,
+cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-311.pyc,,
+cryptography/hazmat/primitives/serialization/base.py,sha256=ikq5MJIwp_oUnjiaBco_PmQwOTYuGi-XkYUYHKy8Vo0,615
+cryptography/hazmat/primitives/serialization/pkcs12.py,sha256=7vVXbiP7qhhvKAHJT_M8-LBZdbpOwrpWRHWxNrNqzXE,4492
+cryptography/hazmat/primitives/serialization/pkcs7.py,sha256=CNzcsuDMyEFMe3EUii4NfJlQzmakB2hLlfRFYObnHRs,11141
+cryptography/hazmat/primitives/serialization/ssh.py,sha256=VKscMrVdYK5B9PQISjjdRMglRvqa_L3sDNm5vdjVHJY,51915
+cryptography/hazmat/primitives/twofactor/__init__.py,sha256=tmMZGB-g4IU1r7lIFqASU019zr0uPp_wEBYcwdDCKCA,258
+cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-311.pyc,,
+cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-311.pyc,,
+cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-311.pyc,,
+cryptography/hazmat/primitives/twofactor/hotp.py,sha256=l1YdRMIhfPIuHKkA66keBDHhNbnBAlh6-O44P-OHIK8,2976
+cryptography/hazmat/primitives/twofactor/totp.py,sha256=v0y0xKwtYrP83ypOo5Ofd441RJLOkaFfjmp554jo5F0,1450
+cryptography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cryptography/utils.py,sha256=Rp7ppg4XIBVVzNQ6XngGndwkICJoYp6FoFOOgTWLJ7g,3925
+cryptography/x509/__init__.py,sha256=uGdiViR7KFnWGoJFVUStt-e_ufomWc87RQBGAZ7dT-4,7980
+cryptography/x509/__pycache__/__init__.cpython-311.pyc,,
+cryptography/x509/__pycache__/base.cpython-311.pyc,,
+cryptography/x509/__pycache__/certificate_transparency.cpython-311.pyc,,
+cryptography/x509/__pycache__/extensions.cpython-311.pyc,,
+cryptography/x509/__pycache__/general_name.cpython-311.pyc,,
+cryptography/x509/__pycache__/name.cpython-311.pyc,,
+cryptography/x509/__pycache__/ocsp.cpython-311.pyc,,
+cryptography/x509/__pycache__/oid.cpython-311.pyc,,
+cryptography/x509/__pycache__/verification.cpython-311.pyc,,
+cryptography/x509/base.py,sha256=3NbbUn9wPruhmoPO7Cl3trc3SrqV2OFIBBE0P2l05mg,37081
+cryptography/x509/certificate_transparency.py,sha256=6HvzAD0dlSQVxy6tnDhGj0-pisp1MaJ9bxQNRr92inI,2261
+cryptography/x509/extensions.py,sha256=R70KkJ_c5NQ6Kx7Rho0sGJ0Rh-bOuBHjVOFSQGRAFCs,67370
+cryptography/x509/general_name.py,sha256=sP_rV11Qlpsk4x3XXGJY_Mv0Q_s9dtjeLckHsjpLQoQ,7836
+cryptography/x509/name.py,sha256=MYCxCSTQTpzhjxFPZaANqJ9fGrhESH73vPkoay8HSWM,14830
+cryptography/x509/ocsp.py,sha256=P6A02msz5pe-IkUFpvxezHvnEHGvPdXiD3S0wsuf4-I,20003
+cryptography/x509/oid.py,sha256=X8EbhkRTLrGuv9vHZSGqPd9zpvRVsonU_joWAL5LLY8,885
+cryptography/x509/verification.py,sha256=alfx3VaTSb2bMz7_7s788oL90vzgHwBjVINssdz0Gv0,796

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.7.0)
+Root-Is-Purelib: false
+Tag: cp39-abi3-manylinux_2_28_x86_64
+

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/license_files/LICENSE

@@ -0,0 +1,3 @@
+This software is made available under the terms of *either* of the licenses
+found in LICENSE.APACHE or LICENSE.BSD. Contributions to cryptography are made
+under the terms of *both* these licenses.

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/license_files/LICENSE.APACHE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

.venv/lib/python3.11/site-packages/cryptography-43.0.3.dist-info/license_files/LICENSE.BSD

@@ -0,0 +1,27 @@
+Copyright (c) Individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of PyCA Cryptography nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/METADATA

@@ -0,0 +1,176 @@
+Metadata-Version: 2.3
+Name: jsonschema
+Version: 4.23.0
+Summary: An implementation of JSON Schema validation for Python
+Project-URL: Homepage, https://github.com/python-jsonschema/jsonschema
+Project-URL: Documentation, https://python-jsonschema.readthedocs.io/
+Project-URL: Issues, https://github.com/python-jsonschema/jsonschema/issues/
+Project-URL: Funding, https://github.com/sponsors/Julian
+Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-jsonschema&utm_medium=referral&utm_campaign=pypi-link
+Project-URL: Changelog, https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst
+Project-URL: Source, https://github.com/python-jsonschema/jsonschema
+Author-email: Julian Berman <[email protected]>
+License: MIT
+License-File: COPYING
+Keywords: data validation,json,json schema,jsonschema,validation
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: File Formats :: JSON
+Classifier: Topic :: File Formats :: JSON :: JSON Schema
+Requires-Python: >=3.8
+Requires-Dist: attrs>=22.2.0
+Requires-Dist: importlib-resources>=1.4.0; python_version < '3.9'
+Requires-Dist: jsonschema-specifications>=2023.03.6
+Requires-Dist: pkgutil-resolve-name>=1.3.10; python_version < '3.9'
+Requires-Dist: referencing>=0.28.4
+Requires-Dist: rpds-py>=0.7.1
+Provides-Extra: format
+Requires-Dist: fqdn; extra == 'format'
+Requires-Dist: idna; extra == 'format'
+Requires-Dist: isoduration; extra == 'format'
+Requires-Dist: jsonpointer>1.13; extra == 'format'
+Requires-Dist: rfc3339-validator; extra == 'format'
+Requires-Dist: rfc3987; extra == 'format'
+Requires-Dist: uri-template; extra == 'format'
+Requires-Dist: webcolors>=1.11; extra == 'format'
+Provides-Extra: format-nongpl
+Requires-Dist: fqdn; extra == 'format-nongpl'
+Requires-Dist: idna; extra == 'format-nongpl'
+Requires-Dist: isoduration; extra == 'format-nongpl'
+Requires-Dist: jsonpointer>1.13; extra == 'format-nongpl'
+Requires-Dist: rfc3339-validator; extra == 'format-nongpl'
+Requires-Dist: rfc3986-validator>0.1.0; extra == 'format-nongpl'
+Requires-Dist: uri-template; extra == 'format-nongpl'
+Requires-Dist: webcolors>=24.6.0; extra == 'format-nongpl'
+Description-Content-Type: text/x-rst
+
+==========
+jsonschema
+==========
+
+|PyPI| |Pythons| |CI| |ReadTheDocs| |Precommit| |Zenodo|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg
+   :alt: PyPI version
+   :target: https://pypi.org/project/jsonschema/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg
+   :alt: Supported Python versions
+   :target: https://pypi.org/project/jsonschema/
+
+.. |CI| image:: https://github.com/python-jsonschema/jsonschema/workflows/CI/badge.svg
+  :alt: Build status
+  :target: https://github.com/python-jsonschema/jsonschema/actions?query=workflow%3ACI
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://python-jsonschema.readthedocs.io/en/stable/
+
+.. |Precommit| image:: https://results.pre-commit.ci/badge/github/python-jsonschema/jsonschema/main.svg
+   :alt: pre-commit.ci status
+   :target: https://results.pre-commit.ci/latest/github/python-jsonschema/jsonschema/main
+
+.. |Zenodo| image:: https://zenodo.org/badge/3072629.svg
+   :alt: Zenodo DOI
+   :target: https://zenodo.org/badge/latestdoi/3072629
+
+
+``jsonschema`` is an implementation of the `JSON Schema <https://json-schema.org>`_ specification for Python.
+
+.. code:: python
+
+    >>> from jsonschema import validate
+
+    >>> # A sample schema, like what we'd get from json.load()
+    >>> schema = {
+    ...     "type" : "object",
+    ...     "properties" : {
+    ...         "price" : {"type" : "number"},
+    ...         "name" : {"type" : "string"},
+    ...     },
+    ... }
+
+    >>> # If no exception is raised by validate(), the instance is valid.
+    >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema)
+
+    >>> validate(
+    ...     instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema,
+    ... )                                   # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    ValidationError: 'Invalid' is not of type 'number'
+
+It can also be used from the command line by installing `check-jsonschema <https://github.com/python-jsonschema/check-jsonschema>`_.
+
+Features
+--------
+
+* Full support for `Draft 2020-12 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft202012Validator>`_, `Draft 2019-09 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft201909Validator>`_, `Draft 7 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft7Validator>`_, `Draft 6 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft6Validator>`_, `Draft 4 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft4Validator>`_ and `Draft 3 <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/validators/#jsonschema.validators.Draft3Validator>`_
+
+* `Lazy validation <https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/protocols/#jsonschema.protocols.Validator.iter_errors>`_ that can iteratively report *all* validation errors.
+
+* `Programmatic querying <https://python-jsonschema.readthedocs.io/en/latest/errors/>`_ of which properties or items failed validation.
+
+
+Installation
+------------
+
+``jsonschema`` is available on `PyPI <https://pypi.org/project/jsonschema/>`_. You can install using `pip <https://pip.pypa.io/en/stable/>`_:
+
+.. code:: bash
+
+    $ pip install jsonschema
+
+
+Extras
+======
+
+Two extras are available when installing the package, both currently related to ``format`` validation:
+
+    * ``format``
+    * ``format-nongpl``
+
+They can be used when installing in order to include additional dependencies, e.g.:
+
+.. code:: bash
+
+    $ pip install jsonschema'[format]'
+
+Be aware that the mere presence of these dependencies – or even the specification of ``format`` checks in a schema – do *not* activate format checks (as per the specification).
+Please read the `format validation documentation <https://python-jsonschema.readthedocs.io/en/latest/validate/#validating-formats>`_ for further details.
+
+About
+-----
+
+I'm Julian Berman.
+
+``jsonschema`` is on `GitHub <https://github.com/python-jsonschema/jsonschema>`_.
+
+Get in touch, via GitHub or otherwise, if you've got something to contribute, it'd be most welcome!
+
+You can also generally find me on Libera (nick: ``Julian``) in various channels, including ``#python``.
+
+If you feel overwhelmingly grateful, you can also `sponsor me <https://github.com/sponsors/Julian/>`_.
+
+And for companies who appreciate ``jsonschema`` and its continued support and growth, ``jsonschema`` is also now supportable via `TideLift <https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-jsonschema&utm_medium=referral&utm_campaign=readme>`_.
+
+
+Release Information
+-------------------
+
+v4.23.0
+=======
+
+* Do not reorder dictionaries (schemas, instances) that are printed as part of validation errors.
+* Declare support for Py3.13

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/RECORD

@@ -0,0 +1,76 @@
+../../../bin/jsonschema,sha256=XHo009U0gdoLuMe818hXZdVGUS_4zdLM86D9zqPvDvA,231
+jsonschema-4.23.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+jsonschema-4.23.0.dist-info/METADATA,sha256=Hd96gAfdO0v5RpFeT25qjyo7PvhASy56F4Jw3FUUTlo,7906
+jsonschema-4.23.0.dist-info/RECORD,,
+jsonschema-4.23.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+jsonschema-4.23.0.dist-info/entry_points.txt,sha256=vO7rX4Fs_xIVJy2pnAtKgTSxfpnozAVQ0DjCmpMxnWE,51
+jsonschema-4.23.0.dist-info/licenses/COPYING,sha256=T5KgFaE8TRoEC-8BiqE0MLTxvHO0Gxa7hGw0Z2bedDk,1057
+jsonschema/__init__.py,sha256=LkPwscySlJ9lTOp7ZB1M7jQ8mbG7-bYG41iBwbZ-o9s,3941
+jsonschema/__main__.py,sha256=iLsZf2upUB3ilBKTlMnyK-HHt2Cnnfkwwxi_c6gLvSA,115
+jsonschema/__pycache__/__init__.cpython-311.pyc,,
+jsonschema/__pycache__/__main__.cpython-311.pyc,,
+jsonschema/__pycache__/_format.cpython-311.pyc,,
+jsonschema/__pycache__/_keywords.cpython-311.pyc,,
+jsonschema/__pycache__/_legacy_keywords.cpython-311.pyc,,
+jsonschema/__pycache__/_types.cpython-311.pyc,,
+jsonschema/__pycache__/_typing.cpython-311.pyc,,
+jsonschema/__pycache__/_utils.cpython-311.pyc,,
+jsonschema/__pycache__/cli.cpython-311.pyc,,
+jsonschema/__pycache__/exceptions.cpython-311.pyc,,
+jsonschema/__pycache__/protocols.cpython-311.pyc,,
+jsonschema/__pycache__/validators.cpython-311.pyc,,
+jsonschema/_format.py,sha256=F_MA52IkrhOIxDqD8x-01bH37mG5nh0kyNrWUSLtWb8,14591
+jsonschema/_keywords.py,sha256=r8_DrqAfn6QLwQnmXEggveiSU-UaIL2p2nuPINelfFc,14949
+jsonschema/_legacy_keywords.py,sha256=2tWuwRPWbYS7EAl8wBIC_rabGuv1J4dfYLqNEPpShhA,15191
+jsonschema/_types.py,sha256=HQ5QD_oL85zF1FSW2v-5rvfYF0967HJdxSR88kzw2mY,5367
+jsonschema/_typing.py,sha256=NZhPhkBOn9INYZk8G69rDeuRamztgXCMLh10z9cfT6g,610
+jsonschema/_utils.py,sha256=ODga3vrJ6K2wMGxerpgn4ipc9q7ZSqBsvwKU4embLEE,10642
+jsonschema/benchmarks/__init__.py,sha256=A0sQrxDBVHSyQ-8ru3L11hMXf3q9gVuB9x_YgHb4R9M,70
+jsonschema/benchmarks/__pycache__/__init__.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/const_vs_enum.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/contains.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/issue232.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/json_schema_test_suite.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/nested_schemas.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/subcomponents.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/unused_registry.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/useless_applicator_schemas.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/useless_keywords.cpython-311.pyc,,
+jsonschema/benchmarks/__pycache__/validator_creation.cpython-311.pyc,,
+jsonschema/benchmarks/const_vs_enum.py,sha256=DVFi3WDqBalZFOibnjpX1uTSr3Rxa2cPgFcowd7Ukrs,830
+jsonschema/benchmarks/contains.py,sha256=gexQoUrCOwECofbt19BeosQZ7WFL6PDdkX49DWwBlOg,786
+jsonschema/benchmarks/issue232.py,sha256=3LLYLIlBGQnVuyyo2iAv-xky5P6PRFHANx4-zIIQOoE,521
+jsonschema/benchmarks/issue232/issue.json,sha256=eaPOZjMRu5u8RpKrsA9uk7ucPZS5tkKG4D_hkOTQ3Hk,117105
+jsonschema/benchmarks/json_schema_test_suite.py,sha256=PvfabpUYcF4_7csYDTcTauED8rnFEGYbdY5RqTXD08s,320
+jsonschema/benchmarks/nested_schemas.py,sha256=mo07dx-CIgmSOI62CNs4g5xu1FzHklLBpkQoDxWYcKs,1892
+jsonschema/benchmarks/subcomponents.py,sha256=fEyiMzsWeK2pd7DEGCuuY-vzGunwhHczRBWEnBRLKIo,1113
+jsonschema/benchmarks/unused_registry.py,sha256=hwRwONc9cefPtYzkoX_TYRO3GyUojriv0-YQaK3vnj0,940
+jsonschema/benchmarks/useless_applicator_schemas.py,sha256=EVm5-EtOEFoLP_Vt2j4SrCwlx05NhPqNuZQ6LIMP1Dc,3342
+jsonschema/benchmarks/useless_keywords.py,sha256=bj_zKr1oVctFlqyZaObCsYTgFjiiNgPzC0hr1Y868mE,867
+jsonschema/benchmarks/validator_creation.py,sha256=UkUQlLAnussnr_KdCIdad6xx2pXxQLmYtsXoiirKeWQ,285
+jsonschema/cli.py,sha256=SGy9JPg02mgXhNxugU8iXhYNivfSjBhKTNAgV90ty-M,8551
+jsonschema/exceptions.py,sha256=RxE2T5xxgg_B6ttR8a3lCbZyh29RUtFe4oZKMoHPBAE,15035
+jsonschema/protocols.py,sha256=7mpZxO1gfRNMCGXwldwsSN3nEugVfIVyKZ_HZgN1vSw,7174
+jsonschema/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+jsonschema/tests/__pycache__/__init__.cpython-311.pyc,,
+jsonschema/tests/__pycache__/_suite.cpython-311.pyc,,
+jsonschema/tests/__pycache__/fuzz_validate.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_cli.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_deprecations.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_exceptions.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_format.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_jsonschema_test_suite.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_types.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_utils.cpython-311.pyc,,
+jsonschema/tests/__pycache__/test_validators.cpython-311.pyc,,
+jsonschema/tests/_suite.py,sha256=QAfBj34zMbJQ5_JJ2ogpiTlw9hQ6Is43dvo_bpS0EdM,8156
+jsonschema/tests/fuzz_validate.py,sha256=fUA7yTJIihaCwJplkUehZeyB84HcXEcqtY5oPJXIO7I,1114
+jsonschema/tests/test_cli.py,sha256=uFMu2YbIfbSDCnykhLL4-VR3-jg1tvQLJn2Bliwp_Bw,28587
+jsonschema/tests/test_deprecations.py,sha256=9VxOCfWzMG1Tg4OD8riU_Znd6HDOQZkepzVgxsdUdU8,15760
+jsonschema/tests/test_exceptions.py,sha256=JgC-E1ZFZK2puVBp35WFRnG8CNOiSWLYtyLjh9IvFKI,22591
+jsonschema/tests/test_format.py,sha256=eVm5SMaWF2lOPO28bPAwNvkiQvHCQKy-MnuAgEchfEc,3188
+jsonschema/tests/test_jsonschema_test_suite.py,sha256=a2saPs2Cwwg0sdRdu-uJ8goSXLbqrS-pC48QJy0K4DE,8674
+jsonschema/tests/test_types.py,sha256=cF51KTDmdsx06MrIc4fXKt0X9fIsVgw5uhT8CamVa8U,6977
+jsonschema/tests/test_utils.py,sha256=sao74o1PyYMxBfqweokQN48CFSS6yhJk5FkCfMJ5PsI,4163
+jsonschema/tests/test_validators.py,sha256=eiaigsZMzHYYsniQ1UPygaS56a1d-_7-9NC4wVXAhzs,87975
+jsonschema/validators.py,sha256=H31FwHdyB7LP5eunxdBrZ9E57hpvozfnRlZaOYy45jU,47045

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.25.0
+Root-Is-Purelib: true
+Tag: py3-none-any

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jsonschema = jsonschema.cli:main

.venv/lib/python3.11/site-packages/jsonschema-4.23.0.dist-info/licenses/COPYING

@@ -0,0 +1,19 @@
+Copyright (c) 2013 Julian Berman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

.venv/lib/python3.11/site-packages/mistral_common/data/mistral_instruct_tokenizer_240323.model.v3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9addc8bdce5988448ae81b729336f43a81262160ae8da760674badab9d4c7d33
+size 587591

.venv/lib/python3.11/site-packages/mistral_common/protocol/embedding/request.py

@@ -0,0 +1,11 @@
+from typing import List, Optional, Union
+
+from pydantic import Field
+
+from mistral_common.base import MistralBase
+
+
+class EmbeddingRequest(MistralBase):
+    input: Union[str, List[str]] = Field(description="Text to embed.")
+    model: str = Field(description="ID of the model to use.")
+    encoding_format: Optional[str] = Field(default="float", description="The format to return the embeddings in.")

.venv/lib/python3.11/site-packages/mistral_common/protocol/embedding/response.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from pydantic import Field
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.base import UsageInfo
+from mistral_common.protocol.utils import random_uuid
+
+
+class EmbeddingObject(MistralBase):
+    object: str = Field(default="embedding", description="The type of the object returned.")
+    embedding: List[float] = Field(description="The type of the object returned.")
+    index: int = Field(description="The index of the embedding in the input text.")
+
+
+class EmbeddingResponse(MistralBase):
+    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
+    object: str = Field(default="list", description="The type of the object returned.")
+    data: List[EmbeddingObject] = Field(description="List of embeddings.")
+    model: str = Field(description="The model used to generate the embeddings.")
+    usage: UsageInfo

.venv/lib/python3.11/site-packages/mistral_common/tokens/instruct/request.py

@@ -0,0 +1,25 @@
+from typing import Generic, List, Optional
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.instruct.messages import ChatMessageType
+from mistral_common.protocol.instruct.tool_calls import ToolType
+
+
+class FIMRequest(MistralBase):
+    """
+    A valid Fill in the Middle completion request to be tokenized
+    """
+
+    prompt: str
+    suffix: Optional[str] = None
+
+
+class InstructRequest(MistralBase, Generic[ChatMessageType, ToolType]):
+    """
+    A valid request to be tokenized
+    """
+
+    messages: List[ChatMessageType]
+    system_prompt: Optional[str] = None
+    available_tools: Optional[List[ToolType]] = None
+    truncate_at_max_tokens: Optional[int] = None

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/base.py

@@ -0,0 +1,200 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Generic, List, Optional, Protocol, Tuple, TypeVar, Union
+
+import numpy as np
+from pydantic import ConfigDict
+
+from mistral_common.base import MistralBase
+from mistral_common.protocol.instruct.messages import (
+    AssistantMessageType,
+    ContentChunk,
+    ImageChunk,
+    ImageURLChunk,
+    UserMessage,
+)
+from mistral_common.protocol.instruct.tool_calls import Tool
+from mistral_common.tokens.instruct.request import FIMRequest, InstructRequest
+
+
+class SpecialTokens(str, Enum):
+    bos = "<s>"
+    eos = "</s>"
+    begin_inst = "[INST]"
+    end_inst = "[/INST]"
+    begin_tools = "[AVAILABLE_TOOLS]"
+    end_tools = "[/AVAILABLE_TOOLS]"
+    begin_tool_results = "[TOOL_RESULTS]"
+    end_tool_results = "[/TOOL_RESULTS]"
+    tool_calls = "[TOOL_CALLS]"
+    img = "[IMG]"
+    img_break = "[IMG_BREAK]"
+    img_end = "[IMG_END]"
+    prefix = "[PREFIX]"
+    middle = "[MIDDLE]"
+    suffix = "[SUFFIX]"
+    begin_system = "[SYSTEM_PROMPT]"
+    end_system = "[/SYSTEM_PROMPT]"
+    begin_tool_content = "[TOOL_CONTENT]"
+
+
+class TokenizerVersion(str, Enum):
+    v1 = "v1"  # vocab_size = 32000
+    v2 = "v2"  # vocab_size = 32768 with special control tokens [INST], [\INST]
+    v3 = "v3"  # vocab_size = 32768 (spm) OR 128000 (tekken) with improved function calling
+    v7 = "v7"  # vocab_size = 32768 (spm) or 128000 (tekken) with improved system prompt and function calling
+
+
+class Tokenized(MistralBase):
+    """
+    A tokenized InstructRequest
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    tokens: List[int]
+    text: Optional[str] = None
+    prefix_ids: Optional[List[int]] = None
+    images: List[np.ndarray] = []
+
+
+class Tokenizer(ABC):
+    @property
+    @abstractmethod
+    def n_words(self) -> int:
+        """Vocabulary size"""
+
+    @abstractmethod
+    def vocab(self) -> List[str]:
+        """All tokens in the vocabulary as strings"""
+
+    @abstractmethod
+    def id_to_piece(self, token_id: int) -> str:
+        """Convert a token id to the token str"""
+
+    @property
+    @abstractmethod
+    def bos_id(self) -> int:
+        """id of the Beginning of String token"""
+
+    @property
+    @abstractmethod
+    def eos_id(self) -> int:
+        """id of the End of String token"""
+
+    @property
+    @abstractmethod
+    def pad_id(self) -> int:
+        """id of the Pad token"""
+
+    @property
+    @abstractmethod
+    def unk_id(self) -> int:
+        """id of the Unk token"""
+
+    @abstractmethod
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        """String to token ids"""
+
+    @abstractmethod
+    def decode(self, t: List[int]) -> str:
+        """Token ids to string"""
+
+    @abstractmethod
+    def get_control_token(self, s: str) -> int:
+        """Get the id of a control token"""
+
+    @property
+    @abstractmethod
+    def version(self) -> TokenizerVersion:
+        """Get the version of the tokenizer"""
+
+    @abstractmethod
+    def to_string(self, tokens: List[int]) -> str:
+        """Convert token ids to string"""
+
+
+InstructRequestType = TypeVar("InstructRequestType", bound=InstructRequest)
+FIMRequestType = TypeVar("FIMRequestType", bound=FIMRequest)
+TokenizedType = TypeVar("TokenizedType", bound=Tokenized)
+
+
+@dataclass
+class ImageEncoding:
+    tokens: List[int]
+    image: np.ndarray
+
+
+@dataclass
+class SpecialImageIDs:
+    img: int
+    img_break: int
+    img_end: int
+
+    @staticmethod
+    def from_tokenizer(tokenizer: "Tokenizer") -> "SpecialImageIDs":
+        return SpecialImageIDs(
+            img=tokenizer.get_control_token(SpecialTokens.img.value),
+            img_break=tokenizer.get_control_token(SpecialTokens.img_break.value),
+            img_end=tokenizer.get_control_token(SpecialTokens.img_end.value),
+        )
+
+
+class MultiModalEncoder(Protocol):
+    def __call__(self, content: Union[ImageChunk, ImageURLChunk]) -> ImageEncoding:
+        """
+        Encode the given content.
+
+        Args:
+            content (ChunkContent): The content to be encoded.
+
+        Returns:
+            ImageEncoding: The encoded image content.
+        """
+        ...
+
+    @property
+    def image_token(self) -> int:
+        ...
+
+
+class InstructTokenizer(Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]):
+    tokenizer: Tokenizer
+    mm_encoder: Optional[MultiModalEncoder]
+
+    def __init__(self, tokenizer: Tokenizer, mm_encoder: Optional[MultiModalEncoder]) -> None:
+        """Init from tokenizer"""
+
+    @abstractmethod
+    def encode_instruct(self, request: InstructRequestType) -> TokenizedType:
+        """Instruct request to Tokenized object"""
+
+    @abstractmethod
+    def decode(self, tokens: List[int]) -> str:
+        """Convert token ids to string"""
+
+    @abstractmethod
+    def encode_fim(self, request: FIMRequestType) -> TokenizedType:
+        """FIM request to Tokenized object"""
+
+    @abstractmethod
+    def encode_user_message(
+        self,
+        message: UserMessage,
+        available_tools: Optional[List[Tool]],
+        is_last: bool,
+        is_first: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        ...
+
+    @abstractmethod
+    def encode_user_content(
+        self,
+        content: Union[str, List[ContentChunk]],
+        is_last: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        ...

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/mistral.py

@@ -0,0 +1,251 @@
+import warnings
+from pathlib import Path
+from typing import Callable, Dict, Generic, List, Optional, Union
+
+from mistral_common.exceptions import (
+    TokenizerException,
+)
+from mistral_common.protocol.instruct.messages import (
+    UATS,
+    AssistantMessageType,
+    SystemMessageType,
+    ToolMessageType,
+    UserMessageType,
+)
+from mistral_common.protocol.instruct.normalize import InstructRequestNormalizer, normalizer_for_tokenizer_version
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+from mistral_common.protocol.instruct.validator import (
+    MistralRequestValidator,
+    MistralRequestValidatorV3,
+    ValidationMode,
+)
+from mistral_common.tokens.instruct.request import FIMRequest
+from mistral_common.tokens.tokenizers.base import (
+    InstructRequest,
+    InstructRequestType,
+    InstructTokenizer,
+    SpecialTokens,
+    TokenizedType,
+    TokenizerVersion,
+)
+from mistral_common.tokens.tokenizers.multimodal import (
+    ImageEncoder,
+    MultimodalConfig,
+    MultiModalEncoder,
+    SpecialImageIDs,
+)
+from mistral_common.tokens.tokenizers.sentencepiece import (
+    InstructTokenizerV1,
+    InstructTokenizerV2,
+    InstructTokenizerV3,
+    InstructTokenizerV7,
+    SentencePieceTokenizer,
+    get_mm_config,
+    is_sentencepiece,
+)
+from mistral_common.tokens.tokenizers.tekken import Tekkenizer, is_tekken
+
+
+def load_mm_encoder(
+    mm_config: MultimodalConfig, tokenizer: Union[Tekkenizer, SentencePieceTokenizer]
+) -> MultiModalEncoder:
+    special_ids = SpecialImageIDs(
+        img=tokenizer.get_control_token(SpecialTokens.img.value),
+        img_break=tokenizer.get_control_token(SpecialTokens.img_break.value),
+        img_end=tokenizer.get_control_token(SpecialTokens.img_end.value),
+    )
+    return ImageEncoder(mm_config, special_ids)
+
+
+class MistralTokenizer(
+    Generic[UserMessageType, AssistantMessageType, ToolMessageType, SystemMessageType, TokenizedType]
+):
+    def __init__(
+        self,
+        instruct_tokenizer: InstructTokenizer[InstructRequest, FIMRequest, TokenizedType, AssistantMessageType],
+        validator: MistralRequestValidator[UserMessageType, AssistantMessageType, ToolMessageType, SystemMessageType],
+        request_normalizer: InstructRequestNormalizer[
+            UserMessageType, AssistantMessageType, ToolMessageType, SystemMessageType, InstructRequestType
+        ],
+    ):
+        self._chat_completion_request_validator = validator
+        self._instruct_request_normalizer = request_normalizer
+        self.instruct_tokenizer = instruct_tokenizer
+
+    @classmethod
+    def _data_path(cls) -> Path:
+        return Path(__file__).parents[2] / "data"
+
+    @classmethod
+    def v1(cls) -> "MistralTokenizer":
+        """open 7B x 8x7B + embed"""
+        return cls.from_file(str(cls._data_path() / "tokenizer.model.v1"), mode=ValidationMode.test)
+
+    @classmethod
+    def v2(cls) -> "MistralTokenizer":
+        """mistral-small // mistral-large"""
+        return cls.from_file(
+            str(cls._data_path() / "mistral_instruct_tokenizer_240216.model.v2"), mode=ValidationMode.test
+        )
+
+    @classmethod
+    def v3(cls, is_tekken: bool = False, is_mm: bool = False) -> "MistralTokenizer":
+        """open-mixtral-8x22B"""
+        if is_tekken and is_mm:
+            tokenizer_name = "tekken_240911.json"
+        elif is_tekken and not is_mm:
+            tokenizer_name = "tekken_240718.json"
+        elif not is_tekken and is_mm:
+            raise ValueError("Multimodal tokenizer is currently only supported for tekken")
+        else:
+            tokenizer_name = "mistral_instruct_tokenizer_240323.model.v3"
+
+        return cls.from_file(str(cls._data_path() / tokenizer_name), mode=ValidationMode.test)
+
+    @classmethod
+    def v7(cls, is_mm: bool = False) -> "MistralTokenizer":
+        """mistral-large 2.1"""
+        if is_mm:
+            return cls.from_file(
+                str(cls._data_path() / "mistral_instruct_tokenizer_241114.model.v7m1"), mode=ValidationMode.test
+            )
+        else:
+            return cls.from_file(
+                str(cls._data_path() / "mistral_instruct_tokenizer_241114.model.v7"), mode=ValidationMode.test
+            )
+
+    @classmethod
+    def from_model(cls, model: str, strict: bool = False) -> "MistralTokenizer":
+        model_name_to_tokenizer_cls: Dict[str, Callable[[], MistralTokenizer]] = {
+            "ministral-8b-2410": lambda: MistralTokenizer.v3(is_tekken=True),
+            "mistral-tiny-2312": MistralTokenizer.v2,
+            "open-mistral-nemo-2407": lambda: MistralTokenizer.v3(is_tekken=True),
+            "mistral-tiny-2407": MistralTokenizer.v3,
+            "mistral-small-2312": MistralTokenizer.v2,
+            "open-mixtral-8x22b-2404": MistralTokenizer.v3,
+            "mistral-small-2402": MistralTokenizer.v2,
+            "mistral-small-2409": lambda: MistralTokenizer.v3(is_tekken=True),
+            "mistral-medium-2312": MistralTokenizer.v1,
+            "mistral-large-2402": MistralTokenizer.v2,
+            "mistral-large-2407": MistralTokenizer.v3,
+            "mistral-large-2411": MistralTokenizer.v7,
+            "pixtral-large-2411": lambda: MistralTokenizer.v7(is_mm=True),
+            "codestral-2405": MistralTokenizer.v3,
+            "codestral-mamba-2407": MistralTokenizer.v3,
+            "pixtral-12b-2409": lambda: MistralTokenizer.v3(is_tekken=True, is_mm=True),
+            # The following are deprecated - only left for backward comp. Delete in >= 1.6.0
+            "open-mistral-7b": MistralTokenizer.v1,
+            "open-mixtral-8x7b": MistralTokenizer.v1,
+            "mistral-embed": MistralTokenizer.v1,
+            "mistral-small-v1": MistralTokenizer.v2,
+            "mistral-large-v1": MistralTokenizer.v2,
+            "mistral-small": MistralTokenizer.v3,
+            "mistral-large": MistralTokenizer.v3,
+            "open-mixtral-8x22b": MistralTokenizer.v3,
+            "codestral-22b": MistralTokenizer.v3,
+            "mistral-nemo": lambda: MistralTokenizer.v3(is_tekken=True),
+            "pixtral": lambda: MistralTokenizer.v3(is_tekken=True, is_mm=True),
+            "pixtral-large": lambda: MistralTokenizer.v7(is_mm=True),
+        }
+
+        if not strict:
+            warnings.warn(
+                "Calling `MistralTokenizer.from_model(..., strict=False)` is deprecated as it can lead to incorrect "
+                "tokenizers. It is strongly recommended to use MistralTokenizer.from_model(..., strict=True)` "
+                "which will become the default in `mistral_common=1.6.0`."
+                "If you are using `mistral_common` for open-sourced model weights, we recommend using "
+                "`MistralTokenizer.from_file('<path/to/tokenizer/file>')` instead.",
+                FutureWarning,
+            )
+
+            # TODO(Delete this code in mistral_common >= 1.6.0
+            # Prefix search the model name mapping
+            for model_name, tokenizer_cls in model_name_to_tokenizer_cls.items():
+                if model_name in model.lower():
+                    return tokenizer_cls()
+
+        if model not in model_name_to_tokenizer_cls:
+            raise TokenizerException(f"Unrecognized model: {model}")
+
+        return model_name_to_tokenizer_cls[model]()
+
+    @classmethod
+    def from_file(
+        cls,
+        tokenizer_filename: str,
+        mode: ValidationMode = ValidationMode.test,
+    ) -> "MistralTokenizer":
+        """
+        Depending on which model we are loading, tokenization and validation might be different. 💩
+        """
+        tokenizer: Union[SentencePieceTokenizer, Tekkenizer]
+
+        if is_tekken(tokenizer_filename):
+            tokenizer = Tekkenizer.from_file(tokenizer_filename)
+            mm_config = tokenizer.multimodal
+        elif is_sentencepiece(tokenizer_filename):
+            tokenizer = SentencePieceTokenizer(tokenizer_filename)
+            mm_config = get_mm_config(tokenizer_filename)
+        else:
+            raise TokenizerException(f"Unrecognized tokenizer file: {tokenizer_filename}")
+
+        mm_encoder = load_mm_encoder(mm_config, tokenizer) if mm_config is not None else None
+
+        request_normalizer = normalizer_for_tokenizer_version(tokenizer.version)
+
+        if tokenizer.version == TokenizerVersion.v1:
+            assert mm_encoder is None, "Tokenizer version needs to be >= v3"
+            return MistralTokenizer(
+                InstructTokenizerV1(tokenizer),
+                validator=MistralRequestValidator(mode=mode),
+                request_normalizer=request_normalizer,
+            )
+        elif tokenizer.version == TokenizerVersion.v2:
+            assert mm_encoder is None, "Tokenizer version needs to be >= v3"
+            return MistralTokenizer(
+                InstructTokenizerV2(tokenizer),
+                validator=MistralRequestValidator(mode=mode),
+                request_normalizer=request_normalizer,
+            )
+        elif tokenizer.version == TokenizerVersion.v3:
+            return MistralTokenizer(
+                InstructTokenizerV3(tokenizer, mm_encoder=mm_encoder),
+                validator=MistralRequestValidatorV3(mode=mode),
+                request_normalizer=request_normalizer,
+            )
+        elif tokenizer.version == TokenizerVersion.v7:
+            return MistralTokenizer(
+                InstructTokenizerV7(tokenizer, mm_encoder=mm_encoder),
+                validator=MistralRequestValidatorV3(mode=mode),
+                request_normalizer=request_normalizer,
+            )
+        else:
+            raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")
+
+        raise TokenizerException(f"Unrecognized tokenizer version: {tokenizer.version}")
+
+    def encode_chat_completion(
+        self, request: ChatCompletionRequest[UATS], max_model_input_len: Optional[int] = None
+    ) -> TokenizedType:
+        validated_request = self._chat_completion_request_validator.validate_request(request)
+
+        if max_model_input_len is None and request.truncate_for_context_length:
+            # the max_model_input_len arg should not be optionnal ;
+            # but this function is used in many small scripts that have no use
+            # for truncation, and don't provide the max model len
+            raise TokenizerException(
+                "encoding a chat completion request with truncation, but no max model len was provided",
+            )
+
+        instruct_request = self._instruct_request_normalizer.from_chat_completion_request(validated_request)
+
+        if request.truncate_for_context_length:
+            instruct_request.truncate_at_max_tokens = max_model_input_len
+
+        return self.instruct_tokenizer.encode_instruct(instruct_request)
+
+    def encode_fim(self, request: FIMRequest) -> TokenizedType:
+        return self.instruct_tokenizer.encode_fim(request)
+
+    def decode(self, tokens: List[int]) -> str:
+        return self.instruct_tokenizer.decode(tokens)

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/multimodal.py

@@ -0,0 +1,172 @@
+import base64
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from io import BytesIO
+from typing import Tuple, Union
+
+import numpy as np
+from PIL import Image
+
+from mistral_common.multimodal import SerializableImage, download_image
+from mistral_common.protocol.instruct.messages import ImageChunk, ImageURLChunk
+from mistral_common.tokens.tokenizers.base import (
+    ImageEncoding,
+    MultiModalEncoder,
+    SpecialImageIDs,
+)
+
+logger = logging.getLogger(__name__)
+
+
+_cv2_installed: bool
+try:
+    import cv2
+
+    _cv2_installed = True
+except ImportError:
+    _cv2_installed = False
+except Exception as e:
+    # cv2 has lots of import problems: https://github.com/opencv/opencv-python/issues/884
+    # for better UX, let's simply skip all errors that might arise from import for now
+    logger.warning(
+        f"Warning: Your installation of OpenCV appears to be broken: {e}."
+        "Please follow the instructions at https://github.com/opencv/opencv-python/issues/884 "
+        "to correct your environment. The import of cv2 has been skipped."
+    )
+
+
+def is_cv2_installed() -> bool:
+    return _cv2_installed
+
+
+def image_from_chunk(chunk: Union[ImageURLChunk, ImageChunk]) -> SerializableImage:
+    """Get a serializable image from a chunk."""
+    if isinstance(chunk, ImageChunk):
+        return chunk.image
+    if chunk.get_url().startswith("data:image"):
+        data = chunk.get_url().split(",")[1]
+        image_data = base64.b64decode(data)
+        return Image.open(BytesIO(image_data))
+    if chunk.get_url().startswith("http"):
+        return download_image(chunk.get_url())
+
+    raise RuntimeError(f"Unsupported image url scheme {chunk.get_url()}")
+
+
+DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)  # RGB
+DATASET_STD = (0.26862954, 0.26130258, 0.27577711)  # RGB
+
+
+# only relevant for spm
+class MultiModalVersion(str, Enum):
+    m1 = "m1"
+
+    @property
+    def config(self) -> "MultimodalConfig":
+        if self.name == "m1":
+            return MultimodalConfig(16, 1024)
+
+        raise NotImplementedError(f"{self.name}")
+
+
+@dataclass
+class MultimodalConfig:
+    image_patch_size: int
+    max_image_size: int
+
+
+def _convert_to_rgb(image: Image.Image) -> Image.Image:
+    """
+    Convert a PIL image to RGB.
+    We ensure transparent background becomes white.
+    """
+    if image.mode == "RGB":
+        return image
+    if image.mode != "RGBA":
+        image = image.convert("RGBA")
+    white_bg: Image.Image = Image.new("RGBA", image.size, "WHITE")
+    white_bg.paste(image, (0, 0), image)
+    return white_bg.convert("RGB")
+
+
+def normalize(
+    np_image: np.ndarray,
+    mean: Tuple[float, float, float],
+    std: Tuple[float, float, float],
+) -> np.ndarray:
+    """
+    Normalize a tensor image with mean and standard deviation.
+
+    Args:
+    image (np.ndarray): Image to be normalized.
+    mean (tuple[float, float, float]): Mean for each channel.
+    std (tuple[float, float, float]): Standard deviation for each channel.
+
+    Returns:
+    np.ndarray: Normalized image with shape (C, H, W).
+    """
+    np_image = np_image / 255.0
+
+    assert len(np_image.shape) == 3, f"{np_image.shape=}"
+    assert np_image.shape[2] == len(mean) == len(std), f"{np_image.shape=}, {mean=}, {std=}"
+
+    np_image = (np_image - mean) / std
+
+    return np_image.transpose(2, 0, 1)
+
+
+def transform_image(image: Image.Image, new_size: Tuple[int, int]) -> np.ndarray:
+    if not is_cv2_installed():
+        raise ImportError("OpenCV is required for this function. Install it with 'pip install mistral_common[opencv]'")
+
+    np_image = cv2.resize(np.array(_convert_to_rgb(image), dtype=np.float32), new_size, interpolation=cv2.INTER_CUBIC)
+    return normalize(np_image, DATASET_MEAN, DATASET_STD)
+
+
+class ImageEncoder(MultiModalEncoder):
+    def __init__(self, mm_config: MultimodalConfig, special_ids: SpecialImageIDs) -> None:
+        self.mm_config = mm_config
+        self.special_ids = special_ids
+
+    def _image_to_num_tokens(self, img: Image.Image) -> Tuple[int, int]:
+        w: Union[int, float]
+        h: Union[int, float]
+
+        w, h = img.size
+        ratio = max(h / self.mm_config.max_image_size, w / self.mm_config.max_image_size)
+        if ratio > 1:
+            w = round(w / ratio)
+            h = round(h / ratio)
+
+        width_tokens = (w - 1) // self.mm_config.image_patch_size + 1
+        height_tokens = (h - 1) // self.mm_config.image_patch_size + 1
+
+        return width_tokens, height_tokens
+
+    def __call__(self, content: Union[ImageChunk, ImageURLChunk]) -> ImageEncoding:
+        """
+        Converts ImageChunks to numpy image arrays and image token ids
+
+        Args:
+        image (ImageChunk, ImageURLChunk): ImageChunk to be converted
+
+        Returns:
+        ImageEncoding containing image token ids and processed image in numpy format
+        """
+        image = image_from_chunk(content)
+        w, h = self._image_to_num_tokens(image)
+        assert w > 0
+        assert h > 0
+        image_tokens = ([self.special_ids.img] * w + [self.special_ids.img_break]) * h
+        image_tokens[-1] = self.special_ids.img_end
+        new_image_size = (
+            w * self.mm_config.image_patch_size,
+            h * self.mm_config.image_patch_size,
+        )
+        processed_image = transform_image(image, new_image_size)
+        return ImageEncoding(tokens=image_tokens, image=processed_image)
+
+    @property
+    def image_token(self) -> int:
+        return self.special_ids.img

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/sentencepiece.py

@@ -0,0 +1,672 @@
+import json
+import logging
+import os
+from abc import abstractmethod
+from functools import cached_property
+from pathlib import Path
+from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Union
+
+import numpy as np
+from sentencepiece import SentencePieceProcessor
+
+from mistral_common.exceptions import TokenizerException
+from mistral_common.protocol.instruct.messages import (
+    AssistantMessage,
+    AssistantMessageType,
+    ContentChunk,
+    SystemMessage,
+    TextChunk,
+    ToolMessage,
+    UserMessage,
+)
+from mistral_common.protocol.instruct.tool_calls import Tool, ToolCall
+from mistral_common.tokens.instruct.request import FIMRequest, InstructRequest
+from mistral_common.tokens.tokenizers.base import (
+    FIMRequestType,
+    InstructRequestType,
+    InstructTokenizer,
+    SpecialTokens,
+    Tokenized,
+    TokenizedType,
+    Tokenizer,
+    TokenizerVersion,
+)
+from mistral_common.tokens.tokenizers.multimodal import MultimodalConfig, MultiModalEncoder, MultiModalVersion
+
+
+def is_sentencepiece(path: Union[str, Path]) -> bool:
+    if isinstance(path, str):
+        path = Path(path)
+
+    instruct_versions = list(TokenizerVersion.__members__)
+    mm_versions = list(MultiModalVersion.__members__) + [""]  # allow no mm version
+    suffixes = [f".model.{v}{m}" for v in instruct_versions for m in mm_versions] + [".model"]
+
+    return path.is_file() and any(path.name.endswith(suffix) for suffix in suffixes)
+
+
+def get_spm_version(tokenizer_filename: str, raise_deprecated: bool = False) -> TokenizerVersion:
+    _version_str = tokenizer_filename.split(".")[-1].split("m")[0]
+    if _version_str == "model":
+        if raise_deprecated:
+            raise TokenizerException(f"Make sure to rename your tokenizer file to end with {tokenizer_filename}.v1.")
+
+        # tokenizer.model => tokenizer.model.v1
+        return TokenizerVersion("v1")
+
+    if _version_str not in TokenizerVersion.__members__:
+        raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")
+
+    return TokenizerVersion(_version_str)
+
+
+def get_mm_config(tokenizer_filename: str) -> Optional[MultimodalConfig]:
+    _version_str = tokenizer_filename.split(".")[-1]
+    if "m" not in _version_str:
+        return None
+
+    _mm_version_str = "m" + _version_str.split("m")[-1]
+
+    if _mm_version_str not in MultiModalVersion.__members__:
+        raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")
+
+    return MultiModalVersion(_mm_version_str).config
+
+
+class SentencePieceTokenizer(Tokenizer):
+    def __init__(self, model_path: str, tokenizer_version: Optional[TokenizerVersion] = None) -> None:
+        self._logger = logging.getLogger(self.__class__.__name__)
+        # reload tokenizer
+        assert os.path.isfile(model_path), model_path
+        self._model = SentencePieceProcessor(model_file=model_path)
+
+        assert self._model.vocab_size() == self._model.get_piece_size()
+        self._vocab = [self._model.id_to_piece(i) for i in range(self.n_words)]
+
+        self._version: TokenizerVersion = tokenizer_version or get_spm_version(model_path, raise_deprecated=False)
+
+        super().__init__()
+
+    @property
+    def version(self) -> TokenizerVersion:
+        return self._version
+
+    def get_control_token(self, s: str) -> int:
+        return self._model.piece_to_id(s)  # type: ignore
+
+    @property
+    def n_words(self) -> int:
+        return self._model.vocab_size()  # type: ignore
+
+    def vocab(self) -> List[str]:
+        return self._vocab
+
+    @property
+    def bos_id(self) -> int:
+        return self._model.bos_id()  # type: ignore
+
+    @property
+    def eos_id(self) -> int:
+        return self._model.eos_id()  # type: ignore
+
+    @cached_property
+    def _control_tokens(self) -> Set[int]:
+        return {tok for tok in range(self.n_words) if self._model.IsControl(tok)}
+
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        assert isinstance(s, str)
+        t: List[int] = self._model.encode(s)
+        if bos:
+            t = [self.bos_id, *t]
+        if eos:
+            t = [*t, self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        return self._model.decode(t)  # type: ignore
+
+    def id_to_piece(self, token_id: int) -> str:
+        return self._model.id_to_piece(token_id)  # type: ignore
+
+    def to_string(self, tokens: List[int]) -> str:
+        """
+        Converts tokens into a string for debugging purposes
+        """
+        text = ""
+        curr_tokens: List[int] = []
+        for tok in tokens:
+            if tok in self._control_tokens:
+                if curr_tokens:
+                    text += "".join([self.id_to_piece(tok) for tok in curr_tokens])
+                    curr_tokens = []
+
+                text += self.id_to_piece(tok)
+
+            else:
+                curr_tokens.append(tok)
+
+        if curr_tokens:
+            text += "".join([self.id_to_piece(tok) for tok in curr_tokens])
+
+        return text
+
+    @property
+    def pad_id(self) -> int:
+        return self._model.pad_id()  # type: ignore
+
+    @property
+    def unk_id(self) -> int:
+        return self._model.unk_id()  # type: ignore
+
+
+class InstructTokenizerBase(
+    InstructTokenizer, Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]
+):
+    def __init__(self, tokenizer: Tokenizer, mm_encoder: Optional[MultiModalEncoder] = None):
+        self.tokenizer = tokenizer
+        self.mm_encoder = mm_encoder
+        super().__init__(tokenizer, mm_encoder)
+
+    def start(self) -> List[int]:
+        return [self.tokenizer.bos_id]
+
+    @staticmethod
+    def find_first_last_user(request: InstructRequest) -> Tuple[int, int]:
+        # find last user message
+        last_user_idx = -1
+        first_user_idx = -1
+        for i, msg in list(enumerate(request.messages)):
+            if isinstance(msg, UserMessage):
+                if first_user_idx == -1:
+                    first_user_idx = i
+                last_user_idx = i
+        return first_user_idx, last_user_idx
+
+    @abstractmethod
+    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> List[int]:
+        raise NotImplementedError("Tool message not implemented")
+
+    @abstractmethod
+    def encode_assistant_message(self, message: AssistantMessageType, is_before_last_user_message: bool) -> List[int]:
+        raise NotImplementedError("Assistant message not implemented")
+
+    def _truncate_for_max_tokens(
+        self,
+        tokenized: List[Optional[List[int]]],
+        messages: List[AssistantMessageType],
+        max_tokens: int,
+        last_user_message_index: int,
+    ) -> None:
+        # Tokenizer ⩽ V3 does not support truncation
+        return
+
+    def encode_instruct(
+        self,
+        request: InstructRequest[AssistantMessageType, Tool],
+    ) -> Tokenized:
+        # init at bos
+        images: List[np.ndarray] = []
+        prefix_ids: Optional[List[int]] = None
+        tokens_list: List[Optional[List[int]]] = []
+
+        # find last user message
+        first_user_idx, last_user_idx = self.find_first_last_user(request)
+        for msg_idx, msg in enumerate(request.messages):
+            if isinstance(msg, UserMessage):
+                new_tokens, new_images = self.encode_user_message(
+                    msg,
+                    request.available_tools,
+                    msg_idx == last_user_idx,
+                    msg_idx == first_user_idx,
+                    system_prompt=request.system_prompt,
+                    force_img_first=True,  # img is always first when providing text/img chunk pair
+                )
+                images.extend(new_images)
+            elif isinstance(msg, ToolMessage):
+                new_tokens = self.encode_tool_message(msg, msg_idx < last_user_idx)
+            elif isinstance(msg, AssistantMessage):
+                new_tokens = self.encode_assistant_message(msg, msg_idx < last_user_idx)
+                if msg_idx == len(request.messages) - 1:
+                    prefix_ids = new_tokens
+            elif isinstance(msg, SystemMessage):
+                new_tokens = self.encode_system_message(msg)
+
+            tokens_list.append(new_tokens)
+
+        if request.truncate_at_max_tokens is not None:
+            self._truncate_for_max_tokens(
+                tokens_list,
+                request.messages,
+                request.truncate_at_max_tokens,
+                last_user_idx,
+            )
+        tokens = self.start()
+
+        for tok in tokens_list:
+            if tok is not None:
+                tokens.extend(tok)
+
+        return Tokenized(
+            tokens=tokens,
+            text=self.tokenizer.to_string(tokens),
+            prefix_ids=prefix_ids,
+            images=images,
+        )
+
+    def decode(self, tokens: List[int]) -> str:
+        return self.tokenizer.decode(tokens)
+
+
+class InstructTokenizerV1(
+    InstructTokenizerBase, Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]
+):
+    def encode_user_message(
+        self,
+        message: UserMessage,
+        available_tools: Optional[List[Tool]],
+        is_last: bool,
+        is_first: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        assert message.content is not None
+        assert isinstance(message.content, str), "Message content must be normalized"
+        assert self.mm_encoder is None, "InstructTokenizerV1 cannot encode images"
+
+        content = ""
+        if is_first and system_prompt:
+            content = system_prompt + "\n\n" + message.content
+        else:
+            content = message.content
+
+        message_txt = f"[INST] {content} [/INST]"
+        curr_tokens, image_tokens = self.encode_user_content(content=message_txt, is_last=False, system_prompt=None)
+        return curr_tokens, image_tokens
+
+    def encode_user_content(
+        self,
+        content: Union[str, List[ContentChunk]],
+        is_last: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        assert isinstance(content, str)
+
+        if is_last and system_prompt:
+            content = system_prompt + "\n\n" + content
+
+        tokens = self.tokenizer.encode(content, bos=False, eos=False)
+        return tokens, []
+
+    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> List[int]:
+        raise TokenizerException("Tools not implemented for tokenizer V1")
+
+    def encode_assistant_message(self, message: AssistantMessageType, is_before_last_user_message: bool) -> List[int]:
+        assert isinstance(message, AssistantMessage), message
+        if message.tool_calls is not None and len(message.tool_calls) > 0:
+            raise TokenizerException("Tools not implemented for tokenizer V1")
+        elif message.content:
+            curr_tokens = self.tokenizer.encode(message.content, bos=False, eos=False)
+        else:
+            raise TokenizerException(f"{message.content} // {message.tool_calls}")
+        if not message.prefix:
+            curr_tokens.append(self.tokenizer.eos_id)
+        return curr_tokens
+
+    def encode_fim(self, request: FIMRequest) -> Tokenized:
+        raise TokenizerException("FIM not available for tokenizer V1")
+
+
+class InstructTokenizerV2(
+    InstructTokenizerV1, Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]
+):
+    def __init__(self, tokenizer: Tokenizer, mm_encoder: Optional[MultiModalEncoder] = None):
+        super().__init__(tokenizer, mm_encoder)
+        self.BEGIN_INST = self.tokenizer.get_control_token(SpecialTokens.begin_inst.value)
+        self.END_INST = self.tokenizer.get_control_token(SpecialTokens.end_inst.value)
+        self.BEGIN_AVAILABLE_TOOLS = self.tokenizer.get_control_token(SpecialTokens.begin_tools.value)
+        self.END_AVAILABLE_TOOLS = self.tokenizer.get_control_token(SpecialTokens.end_tools.value)
+        self.BEGIN_TOOL_RESULTS = self.tokenizer.get_control_token(SpecialTokens.begin_tool_results.value)
+        self.END_TOOL_RESULTS = self.tokenizer.get_control_token(SpecialTokens.end_tool_results.value)
+        self.TOOL_CALLS = self.tokenizer.get_control_token(SpecialTokens.tool_calls.value)
+        self.BOS = self.tokenizer.get_control_token(SpecialTokens.bos.value)
+        self.PREFIX = self.tokenizer.get_control_token(SpecialTokens.prefix.value)
+        self.SUFFIX = self.tokenizer.get_control_token(SpecialTokens.suffix.value)
+
+    def encode_user_message(
+        self,
+        message: UserMessage,
+        available_tools: Optional[List[Tool]],
+        is_last: bool,
+        is_first: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        assert message.content is not None
+        tools_tokens: List[int] = []
+        if is_last and available_tools:
+            tools = [tool.model_dump() for tool in available_tools]
+            tools_json_tokens = self.tokenizer.encode(json.dumps(tools, ensure_ascii=False), bos=False, eos=False)
+            tools_tokens = [
+                self.BEGIN_AVAILABLE_TOOLS,
+                *tools_json_tokens,
+                self.END_AVAILABLE_TOOLS,
+            ]
+
+        tokens, image_tokens = self.encode_user_content(
+            content=message.content,
+            is_last=is_last,
+            system_prompt=system_prompt,
+            force_img_first=force_img_first,
+        )
+
+        prefix_tokens = [*tools_tokens, self.BEGIN_INST]
+        suffix_tokens = [self.END_INST]
+
+        curr_tokens = prefix_tokens + tokens + suffix_tokens
+
+        return curr_tokens, image_tokens
+
+    def _parse_json_content(self, content: str) -> Any:
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            return content
+
+    def _prepare_tool_result(self, tool_message: ToolMessage) -> Dict[str, Any]:
+        """
+        Bit of a hack due to the way tool results are tokenized
+        """
+        assert tool_message.content is not None, "Tool message content cannot be None"
+        return {
+            "name": tool_message.name,
+            "content": self._parse_json_content(tool_message.content),
+        }
+
+    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> List[int]:
+        if is_before_last_user_message:
+            # don't tokenize last tool response before last user msg
+            return []
+
+        # Currently only supports single tool results
+        tool_result_str = json.dumps([self._prepare_tool_result(message)], ensure_ascii=False)
+        curr_tokens = [
+            self.BEGIN_TOOL_RESULTS,
+            *self.tokenizer.encode(tool_result_str, bos=False, eos=False),
+            self.END_TOOL_RESULTS,
+        ]
+        return curr_tokens
+
+    def _prepare_function_call(self, tool_call: ToolCall) -> Dict[str, Any]:
+        """
+        Bit of a hack due to the way function calls are tokenized
+        """
+        return {
+            "name": tool_call.function.name,
+            "arguments": self._parse_json_content(tool_call.function.arguments),
+        }
+
+    def _encode_normal_content_assistant_message(self, message: AssistantMessageType) -> List[int]:
+        assert message.content, f"Assistant message must have content. Got {message}"
+        return self.tokenizer.encode(message.content.rstrip(" "), bos=False, eos=False)
+
+    def _encode_tool_calls_in_assistant_message(self, message: AssistantMessageType) -> List[int]:
+        assert message.tool_calls, f"Assistant message must have tool calls. Got {message}"
+        prepared_tool_calls = []
+        for tool_call in message.tool_calls:
+            prepared_tool_calls.append(self._prepare_function_call(tool_call))
+        tool_call_str = json.dumps(prepared_tool_calls, ensure_ascii=False)
+        curr_tokens = [
+            self.TOOL_CALLS,
+            *self.tokenizer.encode(tool_call_str, bos=False, eos=False),
+        ]
+        return curr_tokens
+
+    def encode_assistant_message(self, message: AssistantMessageType, is_before_last_user_message: bool) -> List[int]:
+        if message.tool_calls:
+            if is_before_last_user_message:
+                # don't tokenize tool call before last user message
+                return []
+            curr_tokens = self._encode_tool_calls_in_assistant_message(message)
+        elif message.content:
+            curr_tokens = self._encode_normal_content_assistant_message(message)
+        else:
+            raise TokenizerException(f"Invalid assistant message: {message.content}")
+        if not message.prefix:
+            curr_tokens.append(self.tokenizer.eos_id)
+        return curr_tokens
+
+    def _encode_infilling(self, text: str) -> List[int]:
+        """
+        Remove prefix space in the case of SentencePieceTokenizers
+        Thanks Fabian !
+        """
+
+        return self.tokenizer.encode("☺" + text, bos=False, eos=False)[2:]
+
+    def encode_fim(self, request: FIMRequest) -> Tokenized:
+        prefix_tokens = self.tokenizer.encode(request.prompt, bos=False, eos=False)
+        suffix_tokens = self._encode_infilling(request.suffix) if request.suffix else []
+        tokens = [
+            self.BOS,
+            self.SUFFIX,
+            *suffix_tokens,
+            self.PREFIX,
+            *prefix_tokens,
+        ]
+        return Tokenized(tokens=tokens, text=self.tokenizer.to_string(tokens))
+
+
+class InstructTokenizerV3(
+    InstructTokenizerV2, Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]
+):
+    """
+    The only difference with V3 tokenizer is that it encodes the tool messages differently
+    """
+
+    def __init__(self, tokenizer: Tokenizer, mm_encoder: Optional[MultiModalEncoder] = None) -> None:
+        super().__init__(tokenizer, mm_encoder=mm_encoder)
+
+    def _prepare_function_call(self, tool_call: ToolCall) -> Dict[str, Any]:
+        function_call = {
+            "name": tool_call.function.name,
+            "arguments": self._parse_json_content(tool_call.function.arguments),
+        }
+
+        if tool_call.id and tool_call.id != "null":
+            function_call["id"] = tool_call.id
+
+        return function_call
+
+    def _prepare_tool_result(self, tool_message: ToolMessage) -> Dict[str, Any]:
+        assert tool_message.content is not None, "Tool message content cannot be None"
+        assert tool_message.tool_call_id is not None, "Tool message has to have the tool call id defined in v3"
+
+        return {
+            "content": self._parse_json_content(tool_message.content),
+            "call_id": tool_message.tool_call_id,
+        }
+
+    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> List[int]:
+        """
+        Same as V2 but tools not wrapped in a list and history is tokenized also
+        """
+        tool_result_str = json.dumps(self._prepare_tool_result(message), ensure_ascii=False)
+        curr_tokens = [
+            self.BEGIN_TOOL_RESULTS,
+            *self.tokenizer.encode(tool_result_str, bos=False, eos=False),
+            self.END_TOOL_RESULTS,
+        ]
+        return curr_tokens
+
+    def encode_assistant_message(self, message: AssistantMessageType, is_before_last_user_message: bool) -> List[int]:
+        """
+        Same as V2 but always encode tool history
+        """
+        return super().encode_assistant_message(message, False)
+
+    def encode_user_content(
+        self,
+        content: Union[str, List[ContentChunk]],
+        is_last: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        if isinstance(content, str):
+            return super().encode_user_content(content, is_last, system_prompt)
+
+        tokens: List[int] = []
+        images: List[np.ndarray] = []
+
+        has_one_img_one_text_first = (
+            len(content) == 2 and isinstance(content[0], TextChunk) and not isinstance(content[1], TextChunk)
+        )
+        if force_img_first and has_one_img_one_text_first:
+            # make sure that if exactly one image and text chunk are passed we force the image chunk to be first
+            content = [content[1], content[0]]
+
+        first_chunk = True
+        for chunk in content:
+            content = ""
+            if first_chunk and is_last and system_prompt:
+                first_chunk = False
+                content = system_prompt + "\n\n"
+            if isinstance(chunk, TextChunk):
+                content += chunk.text
+                tokens.extend(self.tokenizer.encode(content, bos=False, eos=False))
+            else:
+                assert self.mm_encoder is not None, "Make sure to define a multi-modal encoder at init"
+                if content:
+                    tokens.extend(self.tokenizer.encode(content, bos=False, eos=False))
+
+                img_encoding = self.mm_encoder(chunk)
+
+                tokens.extend(img_encoding.tokens)
+                images.append(img_encoding.image)
+
+        return tokens, images
+
+
+class InstructTokenizerV7(InstructTokenizerV3):
+    """
+    The difference with V3 tokenizer is that it encodes the system prompts differently:
+    - in V7 the system prompts are treated as separate SystemMessages
+    - they are no longer prepended to the last user message
+    - they are printed between special tokens
+    Tool call results are encoded as :
+    - [begin tool call] call_id_tokens [tool_content]  content tokens [end tool call]
+    """
+
+    def __init__(self, tokenizer: Tokenizer, mm_encoder: Optional[MultiModalEncoder] = None) -> None:
+        super().__init__(tokenizer, mm_encoder)
+        self.BEGIN_SYSTEM = self.tokenizer.get_control_token(SpecialTokens.begin_system.value)
+        self.END_SYSTEM = self.tokenizer.get_control_token(SpecialTokens.end_system.value)
+        self.BEGIN_TOOL_CONTENT = self.tokenizer.get_control_token(SpecialTokens.begin_tool_content.value)
+
+    def _truncate_for_max_tokens(
+        self,
+        tokenized_messages: List[Optional[List[int]]],
+        messages: List[AssistantMessageType],
+        max_tokens: int,
+        last_user_message_index: int,
+    ) -> None:
+        # drop some messages to fit in max_tokens. Rules:
+        # - don't drop any system messages
+        # - when a user message is dropped, all following assistant|tool message should be dropped until the next
+        #   user message
+        # - we never drop the last message
+        to_drop = sum(len(t) for t in tokenized_messages if t is not None) - max_tokens
+
+        def drop(idx: int) -> None:
+            nonlocal to_drop
+            if isinstance(messages[idx], SystemMessage):
+                # never drop system messages
+                return
+            if idx == last_user_message_index:
+                # never drop the last user message
+                return
+            tok = tokenized_messages[idx]
+            assert tok is not None
+            to_drop -= len(tok)
+            tokenized_messages[idx] = None
+
+        current_idx = 0
+        while to_drop > 0 and current_idx < len(messages):
+            drop(current_idx)
+            current_idx += 1
+            if isinstance(messages[current_idx - 1], UserMessage):
+                # if we just dropped a UserMessage,
+                # also drop everything until the next user message
+                while current_idx < len(messages) and not isinstance(messages[current_idx], UserMessage):
+                    drop(current_idx)
+                    current_idx += 1
+
+        if to_drop > 0:
+            raise TokenizerException("Input couldn't fit in truncate_at_max_token")
+
+    def encode_system_message(self, message: SystemMessage) -> List[int]:
+        assert message.content is not None
+        assert isinstance(message.content, str), "Message content must be normalized"
+        tokens = [
+            self.BEGIN_SYSTEM,
+            *self.tokenizer.encode(message.content, bos=False, eos=False),
+            self.END_SYSTEM,
+        ]
+        return tokens
+
+    def encode_user_message(
+        self,
+        message: UserMessage,
+        available_tools: Optional[List[Tool]],
+        is_last: bool,
+        is_first: bool,
+        system_prompt: Optional[str] = None,
+        force_img_first: bool = False,
+    ) -> Tuple[List[int], List[np.ndarray]]:
+        assert system_prompt is None, "in Tokenizer V7 we don't encode system prompts in user messages"
+        return super().encode_user_message(
+            message,
+            available_tools,
+            is_last=is_last,
+            is_first=is_first,
+            system_prompt=None,
+            force_img_first=force_img_first,
+        )
+
+    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> List[int]:
+        """
+        Same as V3 but tools not wrapped in a list and history is tokenized also
+        """
+        assert message.tool_call_id is not None
+        tool_call_id_tokens = self.tokenizer.encode(message.tool_call_id, bos=False, eos=False)
+        tokens = self.tokenizer.encode(message.content, bos=False, eos=False)
+
+        prefix_tokens = [
+            self.BEGIN_TOOL_RESULTS,
+            *tool_call_id_tokens,
+            self.BEGIN_TOOL_CONTENT,
+        ]
+        curr_tokens = [
+            *prefix_tokens,
+            *tokens,
+            self.END_TOOL_RESULTS,
+        ]
+        return curr_tokens
+
+    def encode_assistant_message(self, message: AssistantMessageType, is_before_last_user_message: bool) -> List[int]:
+        if not message.content and not message.tool_calls:
+            raise TokenizerException(f"Invalid assistant message: {message}")
+        curr_tokens: list = []
+        if message.content:
+            if isinstance(message.content, str):
+                curr_tokens += self._encode_normal_content_assistant_message(message)
+            elif isinstance(message.content, list):
+                curr_tokens += self.encode_content_chunks(
+                    message.content, is_last=False, system_prompt=None, force_img_first=True
+                ).tokens
+        if message.tool_calls:
+            curr_tokens += self._encode_tool_calls_in_assistant_message(message)
+        if not message.prefix:
+            curr_tokens.append(self.tokenizer.eos_id)
+
+        return curr_tokens

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/tekken.py

@@ -0,0 +1,312 @@
+import base64
+import json
+import logging
+from enum import Enum
+from functools import cached_property
+from itertools import groupby
+from pathlib import Path
+from typing import Dict, List, Optional, Type, TypedDict, Union
+
+import tiktoken
+
+from mistral_common.tokens.tokenizers.base import (
+    SpecialTokens,
+    Tokenizer,
+    TokenizerVersion,
+)
+from mistral_common.tokens.tokenizers.multimodal import MultimodalConfig
+
+logger = logging.getLogger(__name__)
+
+
+def is_tekken(path: Union[str, Path]) -> bool:
+    if isinstance(path, str):
+        path = Path(path)
+    return path.is_file() and "tekken" in path.name and path.suffix == ".json"
+
+
+# Formatting specification of the JSON file
+class TokenInfo(TypedDict):
+    rank: int
+    token_bytes: str  # base64 encoded
+    token_str: Optional[str]
+
+
+class TekkenConfig(TypedDict):
+    pattern: str
+    num_vocab_tokens: int
+    default_vocab_size: int
+    default_num_special_tokens: int
+    version: str
+
+
+class ModelData(TypedDict):
+    vocab: List[TokenInfo]
+    config: TekkenConfig
+    version: int
+    type: str
+    multimodal: MultimodalConfig
+
+
+class SpecialTokenPolicy(Enum):
+    """What to do with special tokens when encoding/decoding."""
+
+    IGNORE = 0
+    KEEP = 1
+    RAISE = 2
+
+
+class Tekkenizer(Tokenizer):
+    SPECIAL_TOKENS = (
+        "<unk>",
+        SpecialTokens.bos,
+        SpecialTokens.eos,
+        SpecialTokens.begin_inst,
+        SpecialTokens.end_inst,
+        SpecialTokens.begin_tools,
+        SpecialTokens.end_tools,
+        SpecialTokens.begin_tool_results,
+        SpecialTokens.end_tool_results,
+        SpecialTokens.tool_calls,
+        SpecialTokens.img,
+        "<pad>",
+        SpecialTokens.img_break,
+        SpecialTokens.img_end,
+        SpecialTokens.prefix,
+        SpecialTokens.middle,
+        SpecialTokens.suffix,
+        SpecialTokens.begin_system,
+        SpecialTokens.end_system,
+        SpecialTokens.begin_tool_content,
+    )
+    SPECIAL_TOKEN_TEMPLATE = "<SPECIAL_{id}>"
+
+    # # note that params has a vocab_size field, but it's not used
+
+    def __init__(
+        self,
+        vocab: List[TokenInfo],
+        pattern: str,
+        vocab_size: int,
+        num_special_tokens: int,
+        version: TokenizerVersion,
+        *,
+        name: str = "tekkenizer",
+        _path: Optional[str] = None,
+        mm_config: Optional[MultimodalConfig] = None,
+    ):
+        assert vocab_size <= len(vocab) + num_special_tokens, (
+            vocab_size,
+            len(vocab),
+            num_special_tokens,
+        )
+        self._vocab_size = vocab_size
+        self._path = _path
+
+        special_tokens = list(self.SPECIAL_TOKENS)
+        assert len(special_tokens) == len(set(special_tokens)), f"Special tokens must be unique: {special_tokens}"
+        assert len(special_tokens) < num_special_tokens
+
+        special_filler = [
+            self.SPECIAL_TOKEN_TEMPLATE.format(id=i) for i in range(len(special_tokens), num_special_tokens)
+        ]
+        if special_filler:
+            logger.info(f"Adding special tokens {special_filler[0]}, ..., {special_filler[-1]}")
+        special_tokens = special_tokens + special_filler
+        assert len(set(special_tokens)) == len(special_tokens) == num_special_tokens, special_tokens
+        inner_vocab_size = vocab_size - num_special_tokens
+
+        # reload vocab
+        self._tekken_token2id_nospecial = _reload_mergeable_ranks(vocab, max_vocab=inner_vocab_size)
+        assert set(range(inner_vocab_size)) == set(self._tekken_token2id_nospecial.values()), (
+            inner_vocab_size,
+            self._tekken_token2id_nospecial,
+        )
+
+        self._model = tiktoken.Encoding(
+            name=name,
+            pat_str=pattern,
+            mergeable_ranks=self._tekken_token2id_nospecial,
+            special_tokens={},  # special tokens are handled manually
+        )
+        self._all_special_tokens = special_tokens
+        self._vocab = [self.id_to_piece(i) for i in range(vocab_size)]
+        self._version = version
+        self._special_token_policy = SpecialTokenPolicy.RAISE
+        self._mm_config = mm_config
+
+    @classmethod
+    def from_file(cls: Type["Tekkenizer"], path: Union[str, Path]) -> "Tekkenizer":
+        if isinstance(path, str):
+            path = Path(path)
+        assert path.exists()
+        with open(path, "r") as f:
+            untyped = json.load(f)
+            if mm := untyped.get("multimodal", None):
+                untyped["multimodal"] = MultimodalConfig(**mm)
+            model_data: ModelData = untyped
+
+        _version_str = model_data["config"].get("version")
+        if _version_str not in TokenizerVersion.__members__:
+            raise ValueError(
+                f"Unknown version: {_version_str} in {path}. "
+                f"Make sure to use a valid version string: {list(TokenizerVersion.__members__)}"
+            )
+
+        return cls(
+            vocab=model_data["vocab"],
+            pattern=model_data["config"]["pattern"],
+            vocab_size=model_data["config"]["default_vocab_size"],
+            num_special_tokens=model_data["config"]["default_num_special_tokens"],
+            version=TokenizerVersion(_version_str),
+            name=path.name.replace(".json", ""),
+            _path=str(path),
+            mm_config=model_data.get("multimodal"),
+        )
+
+    @property
+    def multimodal(self) -> Optional[MultimodalConfig]:
+        return self._mm_config
+
+    @multimodal.setter
+    def multimodal(self, value: MultimodalConfig) -> None:
+        raise ValueError("Can only set Multimodal config at init")
+
+    @property
+    def num_special_tokens(self) -> int:
+        return len(self._all_special_tokens)
+
+    @property
+    def n_words(self) -> int:
+        return self._vocab_size
+
+    @property
+    def version(self) -> TokenizerVersion:
+        return self._version
+
+    @property
+    def special_token_policy(self) -> SpecialTokenPolicy:
+        return self._special_token_policy
+
+    @special_token_policy.setter
+    def special_token_policy(self, policy: SpecialTokenPolicy) -> None:
+        self._special_token_policy = policy
+
+    @cached_property
+    def bos_id(self) -> int:
+        return self.SPECIAL_TOKENS.index("<s>")
+
+    @cached_property
+    def eos_id(self) -> int:
+        return self.SPECIAL_TOKENS.index("</s>")
+
+    @cached_property
+    def pad_id(self) -> int:
+        return self.SPECIAL_TOKENS.index("<pad>")
+
+    @cached_property
+    def unk_id(self) -> int:
+        return self.SPECIAL_TOKENS.index("<unk>")
+
+    def vocab(self) -> List[str]:
+        # when returning self._vocab this will collapse
+        # all tokens for which we have a decoding error into
+        # the <?> string. This is bad and results in things
+        # like len(set(vocab)) != len(vocab))
+        # be careful when using self._vocab
+        return self._vocab
+
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        tokens: List[int] = self._model.encode(s)
+        tokens = [t + self.num_special_tokens for t in tokens]
+        if bos:
+            tokens = [self.bos_id, *tokens]
+        if eos:
+            tokens = [*tokens, self.eos_id]
+        return tokens
+
+    def _decode_all(self, tokens: List[int], special_token_policy: SpecialTokenPolicy) -> List[str]:
+        # Lump special and non-special tokens together to minimize calls to decode
+        decoded: List[str] = []
+        for is_special, group in groupby(tokens, lambda t: t < self.num_special_tokens):
+            if is_special:
+                if special_token_policy == SpecialTokenPolicy.RAISE:
+                    raise ValueError(
+                        f"Decoding `tokens` that contain special tokens ({list(group)}) is not allowed. \n"
+                        "Either make sure `tokens` do not include any special tokens or, "
+                        "if you want to decode `tokens` that includes special tokens, "
+                        "change the tokenizer's special token policy to IGNORE or KEEP: \n"
+                        "```\nfrom mistral_common.tokens.tokenizers.mistral import MistralTokenizer"
+                        "\nfrom mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy"
+                        "\n\ntokenizer = MistralTokenizer.v3(is_tekken=True)"
+                        "\ntekken = tokenizer.instruct_tokenizer.tokenizer"
+                        "\ntekken.special_token_policy = SpecialTokenPolicy.IGNORE  # or SpecialTokenPolicy.KEEP"
+                        "\n```"
+                    )
+                elif special_token_policy == SpecialTokenPolicy.KEEP:
+                    decoded.extend(self._all_special_tokens[t] for t in group)
+                elif special_token_policy == SpecialTokenPolicy.IGNORE:
+                    continue
+                # TODO: Could use "tokens_str" from vocab.json
+                # but need to handle null cases.
+            else:
+                decoded.append(self._model.decode([t - self.num_special_tokens for t in group]))
+        return decoded
+
+    def is_byte(self, token_id: int) -> bool:
+        return 0 <= token_id - self.num_special_tokens < 256
+
+    def get_control_token(self, s: str) -> int:
+        try:
+            return self._all_special_tokens.index(s)
+        except ValueError:
+            raise ValueError(f"Unknown control token {s}")
+
+    def decode(self, tokens: List[int]) -> str:
+        return "".join(self._decode_all(tokens, special_token_policy=self._special_token_policy))
+
+    def to_string(self, tokens: List[int]) -> str:
+        return "".join(self._decode_all(tokens, special_token_policy=SpecialTokenPolicy.KEEP))
+
+    def id_to_piece(self, token_id: int) -> str:
+        """convert a token id to its string representation."""
+        return self._decode_all([token_id], special_token_policy=SpecialTokenPolicy.KEEP)[0]
+
+    def id_to_byte_piece(self, token_id: int) -> bytes:
+        """convert a token id to its byte representation."""
+        if token_id < self.num_special_tokens:
+            if self._special_token_policy == SpecialTokenPolicy.KEEP:
+                return self._all_special_tokens[token_id].encode("utf-8")
+            elif self._special_token_policy == SpecialTokenPolicy.RAISE:
+                raise ValueError(f"{token_id} is a special token")
+
+        return self._model.decode_single_token_bytes(token_id - self.num_special_tokens)
+
+
+def _reload_mergeable_ranks(
+    vocab: List[TokenInfo],
+    max_vocab: Union[int, None] = None,
+) -> Dict[bytes, int]:
+    """
+    Reload our tokenizer JSON file and convert it to Tiktoken format.
+    """
+    logger.info(f"Vocab size: {len(vocab)}")
+    if max_vocab is not None:
+        assert len(vocab) >= max_vocab, (len(vocab), max_vocab)
+        vocab = vocab[:max_vocab]
+        logger.info(f"Cutting vocab to first {len(vocab)} tokens.")
+
+    # build ranks
+    ranks: Dict[bytes, int] = {}
+    for i, x in enumerate(vocab):
+        assert x.keys() == {"rank", "token_bytes", "token_str"}
+        assert x["rank"] == i
+        merge = base64.b64decode(x["token_bytes"])
+        assert i >= 256 or merge == bytes([i]), (i, merge)
+        ranks[merge] = x["rank"]
+
+    # sanity check
+    assert len(ranks) == len(vocab)
+    assert set(ranks.values()) == set(range(len(ranks)))
+
+    return ranks

.venv/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/utils.py

@@ -0,0 +1,6 @@
+from typing import Iterator, List
+
+
+def chunks(lst: List[str], chunk_size: int) -> Iterator[List[str]]:
+    for i in range(0, len(lst), chunk_size):
+        yield lst[i : i + chunk_size]