platform/reworkd_platform/services/tokenizer/lifetime.py

import tiktoken
from fastapi import FastAPI

ENCODING_NAME = "cl100k_base"  # gpt-4, gpt-3.5-turbo, text-embedding-ada-002


def init_tokenizer(app: FastAPI) -> None:  # pragma: no cover
    """
    Initialize tokenizer.

    TikToken downloads the encoding on start. It is then
    stored in the state of the application.

    :param app: current application.
    """
    app.state.token_encoding = tiktoken.get_encoding(ENCODING_NAME)