tokenizer_utils.py

from transformers import PreTrainedTokenizerFast

class CryptGPTTokenizer(PreTrainedTokenizerFast):
    @staticmethod
    def clean_up_tokenization(out_string):
        return out_string.replace(' ', "")

CryptGPTTokenizer.register_for_auto_class("AutoTokenizer")