Retrain tokenizer for case sensitive

train_tokenizer.py

@@ -18,7 +18,7 @@ def train_val_files():
         print(f"Number of files {len(data_files)} after adding {path}")
 
     # add_jsonlines_dir(f"{data_dir}/oscar_nl_cleaned")
-    add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*47*.gz")
+    add_jsonlines_dir(f"{data_dir}/c4_cleaned2", "*47*.gz")
     add_jsonlines_dir(f"{data_dir}/nrc_uniq_cleaned_20210223", "*.gz")
     add_jsonlines_dir(f"{data_dir}/nu_uniq_cleaned_20210225", "*.gz")
     random.Random(SEED).shuffle(data_files)
@@ -42,8 +42,6 @@ train, val = train_val_files()
 
 dataset = load_dataset('json', data_files={'train': train, 'validation': val}, split='train')
 
-model_dir = "/t5-small-dutch"  # ${MODEL_DIR}
-
 vocab_size = 32000
 input_sentence_size = None
 tokenizer = SentencePieceUnigramTokenizer(unk_token="<unk>", eos_token="</s>", pad_token="<pad>")