Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "google/mt5-base",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout": 0.2,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.0.dev0",
+  "use_cache": true,
+  "vocab_size": 250112
+}

eval/NLU/all_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6967047747141897,
+    "predict_ex_match_acc_stderr": 0.008429206590221413,
+    "predict_intent_acc": 0.8641560188298588,
+    "predict_intent_acc_stderr": 0.0062826924104243615,
+    "predict_loss": 0.1605614572763443,
+    "predict_runtime": 35.5709,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 83.608,
+    "predict_slot_micro_f1": 0.8051354203306367,
+    "predict_slot_micro_f1_stderr": 0.003306779487846292,
+    "predict_steps_per_second": 2.614
+}

eval/NLU/predict_test_ar_SA_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6422326832548756,
+    "predict_ex_match_acc_stderr": 0.00878974045469875,
+    "predict_intent_acc": 0.8254875588433087,
+    "predict_intent_acc_stderr": 0.006959819569927037,
+    "predict_loss": 0.18552251160144806,
+    "predict_runtime": 40.7492,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 72.983,
+    "predict_slot_micro_f1": 0.7600713012477719,
+    "predict_slot_micro_f1_stderr": 0.0034290701417012625,
+    "predict_steps_per_second": 2.282
+}

eval/NLU/predict_test_de_DE_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6960322797579018,
+    "predict_ex_match_acc_stderr": 0.008434472793379529,
+    "predict_intent_acc": 0.8665097511768661,
+    "predict_intent_acc_stderr": 0.006236501288075228,
+    "predict_loss": 0.15181481838226318,
+    "predict_runtime": 42.3457,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 70.231,
+    "predict_slot_micro_f1": 0.7999301065874541,
+    "predict_slot_micro_f1_stderr": 0.002868514759159988,
+    "predict_steps_per_second": 2.196
+}

eval/NLU/predict_test_en_US_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.7316745124411567,
+    "predict_ex_match_acc_stderr": 0.008124926574292453,
+    "predict_intent_acc": 0.8893745796906524,
+    "predict_intent_acc_stderr": 0.005751741255333137,
+    "predict_loss": 0.13163883984088898,
+    "predict_runtime": 44.5085,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 66.819,
+    "predict_slot_micro_f1": 0.8224364592462753,
+    "predict_slot_micro_f1_stderr": 0.002682990661439905,
+    "predict_steps_per_second": 2.089
+}

eval/NLU/predict_test_es_ES_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.672158708809684,
+    "predict_ex_match_acc_stderr": 0.008607900903395126,
+    "predict_intent_acc": 0.8691997310020175,
+    "predict_intent_acc_stderr": 0.006182920012011209,
+    "predict_loss": 0.15897074341773987,
+    "predict_runtime": 48.4784,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 61.347,
+    "predict_slot_micro_f1": 0.760014179369018,
+    "predict_slot_micro_f1_stderr": 0.002937805653024112,
+    "predict_steps_per_second": 1.918
+}

eval/NLU/predict_test_fr_FR_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6724949562878278,
+    "predict_ex_match_acc_stderr": 0.008605637141063511,
+    "predict_intent_acc": 0.8715534633490248,
+    "predict_intent_acc_stderr": 0.006135327270048997,
+    "predict_loss": 0.14936570823192596,
+    "predict_runtime": 47.795,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 62.224,
+    "predict_slot_micro_f1": 0.76,
+    "predict_slot_micro_f1_stderr": 0.0028830655698127284,
+    "predict_steps_per_second": 1.946
+}

eval/NLU/predict_test_hu_HU_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6866173503698723,
+    "predict_ex_match_acc_stderr": 0.008505980421980473,
+    "predict_intent_acc": 0.8668459986550101,
+    "predict_intent_acc_stderr": 0.006229850189174613,
+    "predict_loss": 0.15636634826660156,
+    "predict_runtime": 41.0061,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 72.526,
+    "predict_slot_micro_f1": 0.7878147561190352,
+    "predict_slot_micro_f1_stderr": 0.003131543166671167,
+    "predict_steps_per_second": 2.268
+}

eval/NLU/predict_test_ko_KR_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6967047747141897,
+    "predict_ex_match_acc_stderr": 0.008429206590221413,
+    "predict_intent_acc": 0.8641560188298588,
+    "predict_intent_acc_stderr": 0.0062826924104243615,
+    "predict_loss": 0.1605614572763443,
+    "predict_runtime": 35.5709,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 83.608,
+    "predict_slot_micro_f1": 0.8051354203306367,
+    "predict_slot_micro_f1_stderr": 0.003306779487846292,
+    "predict_steps_per_second": 2.614
+}

eval/NLU/predict_test_nl_NL_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6973772696704774,
+    "predict_ex_match_acc_stderr": 0.008423919043029776,
+    "predict_intent_acc": 0.8769334229993275,
+    "predict_intent_acc_stderr": 0.0060239712425670154,
+    "predict_loss": 0.14938229322433472,
+    "predict_runtime": 44.4806,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 66.861,
+    "predict_slot_micro_f1": 0.7815729549833597,
+    "predict_slot_micro_f1_stderr": 0.0028633652210475597,
+    "predict_steps_per_second": 2.091
+}

eval/NLU/predict_test_pl_PL_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6577000672494956,
+    "predict_ex_match_acc_stderr": 0.008700553221391915,
+    "predict_intent_acc": 0.875252185608608,
+    "predict_intent_acc_stderr": 0.006059162397252225,
+    "predict_loss": 0.1657099723815918,
+    "predict_runtime": 41.0364,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 72.472,
+    "predict_slot_micro_f1": 0.7381118881118882,
+    "predict_slot_micro_f1_stderr": 0.00335943053411332,
+    "predict_steps_per_second": 2.266
+}

eval/NLU/predict_test_pt_PT_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6879623402824478,
+    "predict_ex_match_acc_stderr": 0.008496016682772936,
+    "predict_intent_acc": 0.8742434431741762,
+    "predict_intent_acc_stderr": 0.006080104292265127,
+    "predict_loss": 0.14546021819114685,
+    "predict_runtime": 46.2953,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 64.24,
+    "predict_slot_micro_f1": 0.776975884527372,
+    "predict_slot_micro_f1_stderr": 0.0028688118075092983,
+    "predict_steps_per_second": 2.009
+}

eval/NLU/predict_test_ru_RU_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.7027572293207801,
+    "predict_ex_match_acc_stderr": 0.008380845398237554,
+    "predict_intent_acc": 0.8769334229993275,
+    "predict_intent_acc_stderr": 0.0060239712425670154,
+    "predict_loss": 0.14948797225952148,
+    "predict_runtime": 42.4467,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 70.064,
+    "predict_slot_micro_f1": 0.78738236319275,
+    "predict_slot_micro_f1_stderr": 0.0031133915518842528,
+    "predict_steps_per_second": 2.191
+}

eval/NLU/predict_test_tr_TR_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6856086079354405,
+    "predict_ex_match_acc_stderr": 0.00851339868573863,
+    "predict_intent_acc": 0.867518493611298,
+    "predict_intent_acc_stderr": 0.006216508296844391,
+    "predict_loss": 0.158187136054039,
+    "predict_runtime": 39.8979,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 74.54,
+    "predict_slot_micro_f1": 0.7798036465638148,
+    "predict_slot_micro_f1_stderr": 0.00325546760623448,
+    "predict_steps_per_second": 2.331
+}

eval/NLU/predict_test_vi_VN_results.json

@@ -0,0 +1,13 @@
+{
+    "predict_ex_match_acc": 0.6556825823806322,
+    "predict_ex_match_acc_stderr": 0.008712761735655685,
+    "predict_intent_acc": 0.8607935440484197,
+    "predict_intent_acc_stderr": 0.0063475876564366525,
+    "predict_loss": 0.13001519441604614,
+    "predict_runtime": 58.5817,
+    "predict_samples": 2974,
+    "predict_samples_per_second": 50.767,
+    "predict_slot_micro_f1": 0.744088281660536,
+    "predict_slot_micro_f1_stderr": 0.002698287735013735,
+    "predict_steps_per_second": 1.588
+}