First commit

.gitattributes

@@ -25,3 +25,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+events.out.tfevents.1610817954.ronak-3 filter=lfs diff=lfs merge=lfs -text
+events.out.tfevents.1610824073.ronak-3 filter=lfs diff=lfs merge=lfs -text
+graph.pbtxt filter=lfs diff=lfs merge=lfs -text
+model.ckpt-1009900.data-00001-of-00002 filter=lfs diff=lfs merge=lfs -text
+model.ckpt-1009900.meta filter=lfs diff=lfs merge=lfs -text

checkpoint

@@ -0,0 +1,3 @@
+model_checkpoint_path: "model.ckpt-1010900"
+all_model_checkpoint_paths: "model.ckpt-1009900"
+all_model_checkpoint_paths: "model.ckpt-1010900"

commands/command

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-38 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_param=init_checkpoint = 'gs://t5-data/pretrained_models/base/model.ckpt-999900' --gin_file=dataset.gin --gin_file=models/bi_v1.gin --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_param=utils.tpu_mesh_shape.model_parallelism = 1 --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=utils.run.train_dataset_fn = @t5.models.mesh_transformer.tsv_dataset_fn --gin_param=tsv_dataset_fn.filename = 'gs://duo-t5/data/query_doc_pairs.train.tsv' --gin_file=learning_rate_schedules/constant_0_001.gin --gin_param=run.train_steps = 1009900 --gin_param=run.save_checkpoints_steps = 10000 --gin_param=tokens_per_batch = 65536

commands/command.1

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-38 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_param=init_checkpoint = 'gs://pongo-bucket/duo-t5/experiments/38/model.ckpt-1009900' --gin_file=dataset.gin --gin_file=models/bi_v1.gin --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_param=utils.tpu_mesh_shape.model_parallelism = 1 --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=utils.run.train_dataset_fn = @t5.models.mesh_transformer.tsv_dataset_fn --gin_param=tsv_dataset_fn.filename = 'gs://pongo-bucket/duo-t5/data/msmarco_med_rod/query_doc_pairs.train.tsv' --gin_file=learning_rate_schedules/constant_0_001.gin --gin_param=run.train_steps = 1010900 --gin_param=run.save_checkpoints_steps = 10000 --gin_param=tokens_per_batch = 65536

commands/command.2

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-1 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1009900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/1/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/1/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.3

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-2 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1010900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/2/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/2/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.4

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-1 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1009900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/5/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/5/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.5

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-2 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1010900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/6/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/6/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.6

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-1 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1009900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/9/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/9/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.7

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-2 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1010900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/10/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/10/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.8

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-2 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1010900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/14/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/14/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

commands/command.9

@@ -0,0 +1 @@
+/home/ronakice/miniconda3/envs/py37/bin/t5_mesh_transformer --tpu=tpu-ronak-1 --gcp_project=justin-project-300310 --tpu_zone=europe-west4-a --model_dir=gs://pongo-bucket/duo-t5/experiments/38 --gin_file=gs://t5-data/pretrained_models/base/operative_config.gin --gin_file=infer.gin --gin_file=beam_search.gin --gin_param=utils.tpu_mesh_shape.tpu_topology = '2x2' --gin_param=infer_checkpoint_step = 1009900 --gin_param=utils.run.sequence_length = {'inputs': 1024, 'targets': 2} --gin_param=Bitransformer.decode.max_decode_length = 2 --gin_param=input_filename = 'gs://pongo-bucket/trec-covid-complete/data/13/duot5_input.txt' --gin_param=output_filename = 'gs://pongo-bucket/trec-covid-complete/data/13/duot5_output.txt' --gin_param=tokens_per_batch = 131072 --gin_param=Bitransformer.decode.beam_size = 1 --gin_param=Bitransformer.decode.temperature = 0.0 --gin_param=Unitransformer.sample_autoregressive.sampling_keep_top_k = -1

events.out.tfevents.1610817954.ronak-3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f73f8e8fc2910f645e93408610369e7ef17022109f26aac33ca22c649024526
+size 71686468

events.out.tfevents.1610824073.ronak-3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb8b26893aa9a659f6a2202aa7403b9b1f48c49f0cda8c506bc056bdf2a89b0a
+size 71689263

graph.pbtxt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e48e738645f83a7efcc38103726a7e06181d534297bc1ff67d4b0b954f740c3
+size 39150190

model.ckpt-1009900.data-00001-of-00002

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:065488b0fe4fc7a893143862d9bbb4329d2d4ef7f2ec97f97c703718cb0da54d
+size 447754240

model.ckpt-1009900.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e6b5c9606a0540c6b1639a4e16da8a32134e442f90c0f32b6dbdd302a52384b
+size 18630025

operative_config.gin

@@ -0,0 +1,332 @@
+import mesh_tensorflow.optimize
+import mesh_tensorflow.transformer.dataset
+import mesh_tensorflow.transformer.learning_rate_schedules
+import mesh_tensorflow.transformer.t2t_vocabulary
+import mesh_tensorflow.transformer.transformer
+import mesh_tensorflow.transformer.transformer_layers
+import mesh_tensorflow.transformer.utils
+import t5.models.mesh_transformer
+
+# Macros:
+# ==============================================================================
+d_ff = 3072
+d_kv = 64
+d_model = 768
+dropout_rate = 0.1
+init_checkpoint = 'gs://pongo-bucket/duo-t5/experiments/38/model.ckpt-1009900'
+inputs_length = 512
+MIXTURE_NAME = 'all_mix'
+num_heads = 12
+num_layers = 12
+targets_length = 512
+tokens_per_batch = 65536
+
+# Parameters for adafactor_decay_rate_pow:
+# ==============================================================================
+adafactor_decay_rate_pow.offset = 0
+
+# Parameters for AdafactorOptimizer:
+# ==============================================================================
+AdafactorOptimizer.beta1 = 0.0
+AdafactorOptimizer.clipping_threshold = 1.0
+AdafactorOptimizer.decay_rate = None
+AdafactorOptimizer.epsilon1 = 1e-30
+AdafactorOptimizer.epsilon2 = 0.001
+AdafactorOptimizer.factored = True
+AdafactorOptimizer.min_dim_size_to_factor = 128
+AdafactorOptimizer.multiply_by_parameter_scale = True
+
+# Parameters for Bitransformer:
+# ==============================================================================
+Bitransformer.shared_embedding = True
+
+# Parameters for constant_learning_rate:
+# ==============================================================================
+constant_learning_rate.learning_rate = 0.001
+
+# Parameters for decoder/DenseReluDense:
+# ==============================================================================
+decoder/DenseReluDense.activation = 'relu'
+decoder/DenseReluDense.dropout_rate = %dropout_rate
+decoder/DenseReluDense.hidden_size = %d_ff
+decoder/DenseReluDense.use_bias = False
+
+# Parameters for encoder/DenseReluDense:
+# ==============================================================================
+encoder/DenseReluDense.activation = 'relu'
+encoder/DenseReluDense.dropout_rate = %dropout_rate
+encoder/DenseReluDense.hidden_size = %d_ff
+encoder/DenseReluDense.use_bias = False
+
+# Parameters for enc_dec_attention:
+# ==============================================================================
+# None.
+
+# Parameters for enc_dec_attention_bias:
+# ==============================================================================
+# None.
+
+# Parameters for decoder/EncDecAttention:
+# ==============================================================================
+decoder/EncDecAttention.relative_attention_type = None
+
+# Parameters for get_variable_dtype:
+# ==============================================================================
+get_variable_dtype.activation_dtype = 'bfloat16'
+
+# Parameters for get_vocab_embedding_cls:
+# ==============================================================================
+# None.
+
+# Parameters for get_vocabulary:
+# ==============================================================================
+get_vocabulary.mixture_or_task_name = %MIXTURE_NAME
+
+# Parameters for init_checkpoint_variable_mapping:
+# ==============================================================================
+init_checkpoint_variable_mapping.mapping_fn = None
+
+# Parameters for decoder/LayerStack:
+# ==============================================================================
+decoder/LayerStack.dropout_rate = %dropout_rate
+decoder/LayerStack.norm_epsilon = 1e-06
+decoder/LayerStack.recompute_grads = False
+decoder/LayerStack.sublayers_final = \
+    [@transformer.sublayer_rms_norm, @transformer.sublayer_dropout]
+decoder/LayerStack.sublayers_initial = [@transformer.sublayer_dropout]
+decoder/LayerStack.sublayers_per_layer = \
+    [@transformer.sublayer_rms_norm,
+     @transformer.sublayer_call_layer,
+     @transformer.sublayer_dropout,
+     @transformer.sublayer_residual]
+
+# Parameters for encoder/LayerStack:
+# ==============================================================================
+encoder/LayerStack.dropout_rate = %dropout_rate
+encoder/LayerStack.norm_epsilon = 1e-06
+encoder/LayerStack.recompute_grads = False
+encoder/LayerStack.sublayers_final = \
+    [@transformer.sublayer_rms_norm, @transformer.sublayer_dropout]
+encoder/LayerStack.sublayers_initial = [@transformer.sublayer_dropout]
+encoder/LayerStack.sublayers_per_layer = \
+    [@transformer.sublayer_rms_norm,
+     @transformer.sublayer_call_layer,
+     @transformer.sublayer_dropout,
+     @transformer.sublayer_residual]
+
+# Parameters for make_bitransformer:
+# ==============================================================================
+make_bitransformer.decoder_name = 'decoder'
+make_bitransformer.encoder_name = 'encoder'
+
+# Parameters for decoder/make_layer_stack:
+# ==============================================================================
+decoder/make_layer_stack.block_scope = True
+decoder/make_layer_stack.layers = \
+    [@mesh_tensorflow.transformer.transformer_layers.SelfAttention,
+     @mesh_tensorflow.transformer.transformer_layers.EncDecAttention,
+     @mesh_tensorflow.transformer.transformer_layers.DenseReluDense]
+decoder/make_layer_stack.num_layers = %num_layers
+
+# Parameters for encoder/make_layer_stack:
+# ==============================================================================
+encoder/make_layer_stack.block_scope = True
+encoder/make_layer_stack.layers = \
+    [@mesh_tensorflow.transformer.transformer_layers.SelfAttention,
+     @mesh_tensorflow.transformer.transformer_layers.DenseReluDense]
+encoder/make_layer_stack.num_layers = %num_layers
+
+# Parameters for pack_dataset:
+# ==============================================================================
+pack_dataset.use_custom_ops = False
+
+# Parameters for pack_or_pad:
+# ==============================================================================
+pack_or_pad.ensure_eos = False
+pack_or_pad.feature_keys = None
+pack_or_pad.pack = True
+
+# Parameters for packed_parallel_tsv_dataset:
+# ==============================================================================
+packed_parallel_tsv_dataset.batch_size = None
+packed_parallel_tsv_dataset.max_encoded_len = 0
+
+# Parameters for rewrite_stack_variables:
+# ==============================================================================
+rewrite_stack_variables.max_combined_variable_size = 536870912
+
+# Parameters for run:
+# ==============================================================================
+run.autostack = True
+run.batch_size = ('tokens_per_batch', %tokens_per_batch)
+run.dataset_split = 'train'
+run.ensemble_inputs = None
+run.eval_checkpoint_step = None
+run.eval_dataset_fn = None
+run.eval_summary_dir = None
+run.export_checkpoint_step = None
+run.export_path = ''
+run.init_checkpoint = %init_checkpoint
+run.iterations_per_loop = 100
+run.keep_checkpoint_max = None
+run.layout_rules = \
+    'ensemble:ensemble,batch:batch,d_ff:model,heads:model,vocab:model,experts:batch'
+run.learning_rate_schedule = @learning_rate_schedules.constant_learning_rate
+run.mesh_devices = None
+run.mesh_shape = @mesh_tensorflow.transformer.utils.tpu_mesh_shape()
+run.mode = 'train'
+run.model_type = 'bitransformer'
+run.optimizer = @optimize.AdafactorOptimizer
+run.output_eval_examples = True
+run.perplexity_eval_steps = 10
+run.predict_fn = None
+run.save_checkpoints_steps = 10000
+run.sequence_length = {'inputs': %inputs_length, 'targets': %targets_length}
+run.skip_seen_data = False
+run.total_run_steps = None
+run.train_dataset_fn = @t5.models.mesh_transformer.tsv_dataset_fn
+run.train_steps = 1010900
+run.variable_filter = None
+
+# Parameters for decoder/SelfAttention:
+# ==============================================================================
+decoder/SelfAttention.attention_func = None
+decoder/SelfAttention.attention_kwargs = None
+decoder/SelfAttention.combine_dims = True
+decoder/SelfAttention.dropout_rate = %dropout_rate
+decoder/SelfAttention.fold_scaling_into_initializer = True
+decoder/SelfAttention.keep_query_heads_dims = False
+decoder/SelfAttention.key_value_size = %d_kv
+decoder/SelfAttention.num_heads = %num_heads
+decoder/SelfAttention.num_memory_heads = 0
+decoder/SelfAttention.relative_attention_num_buckets = 32
+decoder/SelfAttention.relative_attention_type = 'bias_shared'
+decoder/SelfAttention.shared_kv = False
+
+# Parameters for encoder/SelfAttention:
+# ==============================================================================
+encoder/SelfAttention.attention_func = None
+encoder/SelfAttention.attention_kwargs = None
+encoder/SelfAttention.combine_dims = True
+encoder/SelfAttention.dropout_rate = %dropout_rate
+encoder/SelfAttention.fold_scaling_into_initializer = True
+encoder/SelfAttention.keep_query_heads_dims = False
+encoder/SelfAttention.key_value_size = %d_kv
+encoder/SelfAttention.num_heads = %num_heads
+encoder/SelfAttention.num_memory_heads = 0
+encoder/SelfAttention.relative_attention_num_buckets = 32
+encoder/SelfAttention.relative_attention_type = 'bias_shared'
+encoder/SelfAttention.shared_kv = False
+
+# Parameters for serialize_num_microbatches:
+# ==============================================================================
+serialize_num_microbatches.tokens_per_microbatch_per_replica = 8192
+
+# Parameters for SimdMeshImpl:
+# ==============================================================================
+SimdMeshImpl.allreduce_in_bfloat16_max_group_size = 8
+
+# Parameters for sublayer_call_layer:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_dropout:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_legacy_dropout:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_legacy_final_rms_norm:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_legacy_rms_norm:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_mask_padding:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_residual:
+# ==============================================================================
+# None.
+
+# Parameters for sublayer_rms_norm:
+# ==============================================================================
+sublayer_rms_norm.epsilon = 1e-06
+
+# Parameters for tpu_estimator_model_fn:
+# ==============================================================================
+tpu_estimator_model_fn.hierarchical_tiling_spec = None
+tpu_estimator_model_fn.init_variable_filter = ''
+tpu_estimator_model_fn.model_info_file = None
+tpu_estimator_model_fn.outer_batch_size = 1
+tpu_estimator_model_fn.tpu_summaries = False
+
+# Parameters for tpu_mesh_shape:
+# ==============================================================================
+tpu_mesh_shape.ensemble_parallelism = None
+tpu_mesh_shape.model_parallelism = 1
+tpu_mesh_shape.tpu_topology = '2x2'
+
+# Parameters for train_model:
+# ==============================================================================
+train_model.dataset_split = 'train'
+train_model.seen_data_init_step = 0
+
+# Parameters for tsv_dataset_fn:
+# ==============================================================================
+tsv_dataset_fn.filename = \
+    'gs://pongo-bucket/duo-t5/data/msmarco_med_rod/query_doc_pairs.train.tsv'
+tsv_dataset_fn.shuffle_buffer_size = 10000
+
+# Parameters for unit_scaling_convention:
+# ==============================================================================
+unit_scaling_convention.value = False
+
+# Parameters for decoder/Unitransformer:
+# ==============================================================================
+decoder/Unitransformer.d_model = %d_model
+decoder/Unitransformer.ensemble = None
+decoder/Unitransformer.input_full_attention = False
+decoder/Unitransformer.label_smoothing = 0.0
+decoder/Unitransformer.loss_denominator = 233472
+decoder/Unitransformer.loss_fn = None
+decoder/Unitransformer.loss_on_targets_only = False
+decoder/Unitransformer.max_length = 512
+decoder/Unitransformer.positional_embedding = False
+decoder/Unitransformer.shared_embedding_and_softmax_weights = True
+decoder/Unitransformer.sinusoid_positional_embedding = False
+decoder/Unitransformer.token_dropout_rate = 0.0
+decoder/Unitransformer.vocab_divisor = 128
+decoder/Unitransformer.z_loss = 0.0001
+
+# Parameters for encoder/Unitransformer:
+# ==============================================================================
+encoder/Unitransformer.d_model = %d_model
+encoder/Unitransformer.ensemble = None
+encoder/Unitransformer.input_full_attention = False
+encoder/Unitransformer.label_smoothing = 0.0
+encoder/Unitransformer.loss_denominator = None
+encoder/Unitransformer.loss_fn = None
+encoder/Unitransformer.loss_on_targets_only = False
+encoder/Unitransformer.max_length = 512
+encoder/Unitransformer.positional_embedding = False
+encoder/Unitransformer.shared_embedding_and_softmax_weights = True
+encoder/Unitransformer.sinusoid_positional_embedding = False
+encoder/Unitransformer.token_dropout_rate = 0.0
+encoder/Unitransformer.vocab_divisor = 128
+encoder/Unitransformer.z_loss = 0.0001
+
+# Parameters for VarianceScalingInitializer:
+# ==============================================================================
+VarianceScalingInitializer.distribution = 'normal'
+VarianceScalingInitializer.mode = 'fan_in'
+VarianceScalingInitializer.scale = 1.0
+
+# Parameters for VocabEmbedding:
+# ==============================================================================
+VocabEmbedding.scale_variable_like_classifier_weights = False