{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Comparing TensorFlow (original) and PyTorch models\n", "\n", "You can use this small notebook to check the conversion of the model's weights from the TensorFlow model to the PyTorch model. In the following, we compare the weights of the last layer on a simple example (in `input.txt`) but both models returns all the hidden layers so you can check every stage of the model.\n", "\n", "To run this notebook, follow these instructions:\n", "- make sure that your Python environment has both TensorFlow and PyTorch installed,\n", "- download the original TensorFlow implementation,\n", "- download a pre-trained TensorFlow model as indicaded in the TensorFlow implementation readme,\n", "- run the script `convert_tf_checkpoint_to_pytorch.py` as indicated in the `README` to convert the pre-trained TensorFlow model to PyTorch.\n", "\n", "If needed change the relative paths indicated in this notebook (at the beggining of Sections 1 and 2) to point to the relevent models and code." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:26.999106Z", "start_time": "2018-11-16T10:02:26.985709Z" } }, "outputs": [], "source": [ "import os\n", "os.chdir('../')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1/ TensorFlow code" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:27.664528Z", "start_time": "2018-11-16T10:02:27.651019Z" } }, "outputs": [], "source": [ "original_tf_inplem_dir = \"./tensorflow_code/\"\n", "model_dir = \"../google_models/uncased_L-12_H-768_A-12/\"\n", "\n", "vocab_file = model_dir + \"vocab.txt\"\n", "bert_config_file = model_dir + \"bert_config.json\"\n", "init_checkpoint = model_dir + \"bert_model.ckpt\"\n", "\n", "input_file = \"./samples/input.txt\"\n", "max_seq_length = 128\n", "max_predictions_per_seq = 20\n", "\n", "masked_lm_positions = [6]" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:30.202182Z", "start_time": "2018-11-16T10:02:28.112570Z" } }, "outputs": [], "source": [ "import importlib.util\n", "import sys\n", "import tensorflow as tf\n", "import pytorch_transformers as ppb\n", "\n", "def del_all_flags(FLAGS):\n", " flags_dict = FLAGS._flags() \n", " keys_list = [keys for keys in flags_dict] \n", " for keys in keys_list:\n", " FLAGS.__delattr__(keys)\n", "\n", "del_all_flags(tf.flags.FLAGS)\n", "import tensorflow_code.extract_features as ef\n", "del_all_flags(tf.flags.FLAGS)\n", "import tensorflow_code.modeling as tfm\n", "del_all_flags(tf.flags.FLAGS)\n", "import tensorflow_code.tokenization as tft\n", "del_all_flags(tf.flags.FLAGS)\n", "import tensorflow_code.run_pretraining as rp\n", "del_all_flags(tf.flags.FLAGS)\n", "import tensorflow_code.create_pretraining_data as cpp" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:30.238027Z", "start_time": "2018-11-16T10:02:30.204943Z" }, "code_folding": [ 15 ] }, "outputs": [], "source": [ "import re\n", "class InputExample(object):\n", " \"\"\"A single instance example.\"\"\"\n", "\n", " def __init__(self, tokens, segment_ids, masked_lm_positions,\n", " masked_lm_labels, is_random_next):\n", " self.tokens = tokens\n", " self.segment_ids = segment_ids\n", " self.masked_lm_positions = masked_lm_positions\n", " self.masked_lm_labels = masked_lm_labels\n", " self.is_random_next = is_random_next\n", " def __repr__(self):\n", " return '\\n'.join(k + \":\" + str(v) for k, v in self.__dict__.items())\n", "\n", "\n", "def read_examples(input_file, tokenizer, masked_lm_positions):\n", " \"\"\"Read a list of `InputExample`s from an input file.\"\"\"\n", " examples = []\n", " unique_id = 0\n", " with tf.gfile.GFile(input_file, \"r\") as reader:\n", " while True:\n", " line = reader.readline()\n", " if not line:\n", " break\n", " line = line.strip()\n", " text_a = None\n", " text_b = None\n", " m = re.match(r\"^(.*) \\|\\|\\| (.*)$\", line)\n", " if m is None:\n", " text_a = line\n", " else:\n", " text_a = m.group(1)\n", " text_b = m.group(2)\n", " tokens_a = tokenizer.tokenize(text_a)\n", " tokens_b = None\n", " if text_b:\n", " tokens_b = tokenizer.tokenize(text_b)\n", " tokens = tokens_a + tokens_b\n", " masked_lm_labels = []\n", " for m_pos in masked_lm_positions:\n", " masked_lm_labels.append(tokens[m_pos])\n", " tokens[m_pos] = '[MASK]'\n", " examples.append(\n", " InputExample(\n", " tokens = tokens,\n", " segment_ids = [0] * len(tokens_a) + [1] * len(tokens_b),\n", " masked_lm_positions = masked_lm_positions,\n", " masked_lm_labels = masked_lm_labels,\n", " is_random_next = False))\n", " unique_id += 1\n", " return examples" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:30.304018Z", "start_time": "2018-11-16T10:02:30.240189Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tokens:['who', 'was', 'jim', 'henson', '?', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer']\n", "segment_ids:[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]\n", "masked_lm_positions:[6]\n", "masked_lm_labels:['henson']\n", "is_random_next:False\n" ] } ], "source": [ "bert_config = tfm.BertConfig.from_json_file(bert_config_file)\n", "tokenizer = ppb.BertTokenizer(\n", " vocab_file=vocab_file, do_lower_case=True)\n", "examples = read_examples(input_file, tokenizer, masked_lm_positions=masked_lm_positions)\n", "\n", "print(examples[0])" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:33.324167Z", "start_time": "2018-11-16T10:02:33.291909Z" }, "code_folding": [ 16 ] }, "outputs": [], "source": [ "class InputFeatures(object):\n", " \"\"\"A single set of features of data.\"\"\"\n", "\n", " def __init__(self, input_ids, input_mask, segment_ids, masked_lm_positions,\n", " masked_lm_ids, masked_lm_weights, next_sentence_label):\n", " self.input_ids = input_ids\n", " self.input_mask = input_mask\n", " self.segment_ids = segment_ids\n", " self.masked_lm_positions = masked_lm_positions\n", " self.masked_lm_ids = masked_lm_ids\n", " self.masked_lm_weights = masked_lm_weights\n", " self.next_sentence_labels = next_sentence_label\n", "\n", " def __repr__(self):\n", " return '\\n'.join(k + \":\" + str(v) for k, v in self.__dict__.items())\n", "\n", "def pretraining_convert_examples_to_features(instances, tokenizer, max_seq_length,\n", " max_predictions_per_seq):\n", " \"\"\"Create TF example files from `TrainingInstance`s.\"\"\"\n", " features = []\n", " for (inst_index, instance) in enumerate(instances):\n", " input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)\n", " input_mask = [1] * len(input_ids)\n", " segment_ids = list(instance.segment_ids)\n", " assert len(input_ids) <= max_seq_length\n", "\n", " while len(input_ids) < max_seq_length:\n", " input_ids.append(0)\n", " input_mask.append(0)\n", " segment_ids.append(0)\n", "\n", " assert len(input_ids) == max_seq_length\n", " assert len(input_mask) == max_seq_length\n", " assert len(segment_ids) == max_seq_length\n", "\n", " masked_lm_positions = list(instance.masked_lm_positions)\n", " masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels)\n", " masked_lm_weights = [1.0] * len(masked_lm_ids)\n", "\n", " while len(masked_lm_positions) < max_predictions_per_seq:\n", " masked_lm_positions.append(0)\n", " masked_lm_ids.append(0)\n", " masked_lm_weights.append(0.0)\n", "\n", " next_sentence_label = 1 if instance.is_random_next else 0\n", "\n", " features.append(\n", " InputFeatures(input_ids, input_mask, segment_ids,\n", " masked_lm_positions, masked_lm_ids,\n", " masked_lm_weights, next_sentence_label))\n", "\n", " if inst_index < 5:\n", " tf.logging.info(\"*** Example ***\")\n", " tf.logging.info(\"tokens: %s\" % \" \".join(\n", " [str(x) for x in instance.tokens]))\n", " tf.logging.info(\"features: %s\" % str(features[-1]))\n", " return features" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:34.185367Z", "start_time": "2018-11-16T10:02:34.155046Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:*** Example ***\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:34 - INFO - tensorflow - *** Example ***\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:tokens: who was jim henson ? jim [MASK] was a puppet ##eer\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:34 - INFO - tensorflow - tokens: who was jim henson ? jim [MASK] was a puppet ##eer\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:features: input_ids:[2040, 2001, 3958, 27227, 1029, 3958, 103, 2001, 1037, 13997, 11510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "input_mask:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "segment_ids:[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_positions:[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_ids:[27227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_weights:[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n", "next_sentence_labels:0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:34 - INFO - tensorflow - features: input_ids:[2040, 2001, 3958, 27227, 1029, 3958, 103, 2001, 1037, 13997, 11510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "input_mask:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "segment_ids:[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_positions:[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_ids:[27227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "masked_lm_weights:[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n", "next_sentence_labels:0\n" ] } ], "source": [ "features = pretraining_convert_examples_to_features(\n", " instances=examples, max_seq_length=max_seq_length, \n", " max_predictions_per_seq=max_predictions_per_seq, tokenizer=tokenizer)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:34.912005Z", "start_time": "2018-11-16T10:02:34.882111Z" } }, "outputs": [], "source": [ "def input_fn_builder(features, seq_length, max_predictions_per_seq, tokenizer):\n", " \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n", "\n", " all_input_ids = []\n", " all_input_mask = []\n", " all_segment_ids = []\n", " all_masked_lm_positions = []\n", " all_masked_lm_ids = []\n", " all_masked_lm_weights = []\n", " all_next_sentence_labels = []\n", "\n", " for feature in features:\n", " all_input_ids.append(feature.input_ids)\n", " all_input_mask.append(feature.input_mask)\n", " all_segment_ids.append(feature.segment_ids)\n", " all_masked_lm_positions.append(feature.masked_lm_positions)\n", " all_masked_lm_ids.append(feature.masked_lm_ids)\n", " all_masked_lm_weights.append(feature.masked_lm_weights)\n", " all_next_sentence_labels.append(feature.next_sentence_labels)\n", "\n", " def input_fn(params):\n", " \"\"\"The actual input function.\"\"\"\n", " batch_size = params[\"batch_size\"]\n", "\n", " num_examples = len(features)\n", "\n", " # This is for demo purposes and does NOT scale to large data sets. We do\n", " # not use Dataset.from_generator() because that uses tf.py_func which is\n", " # not TPU compatible. The right way to load data is with TFRecordReader.\n", " d = tf.data.Dataset.from_tensor_slices({\n", " \"input_ids\":\n", " tf.constant(\n", " all_input_ids, shape=[num_examples, seq_length],\n", " dtype=tf.int32),\n", " \"input_mask\":\n", " tf.constant(\n", " all_input_mask,\n", " shape=[num_examples, seq_length],\n", " dtype=tf.int32),\n", " \"segment_ids\":\n", " tf.constant(\n", " all_segment_ids,\n", " shape=[num_examples, seq_length],\n", " dtype=tf.int32),\n", " \"masked_lm_positions\":\n", " tf.constant(\n", " all_masked_lm_positions,\n", " shape=[num_examples, max_predictions_per_seq],\n", " dtype=tf.int32),\n", " \"masked_lm_ids\":\n", " tf.constant(\n", " all_masked_lm_ids,\n", " shape=[num_examples, max_predictions_per_seq],\n", " dtype=tf.int32),\n", " \"masked_lm_weights\":\n", " tf.constant(\n", " all_masked_lm_weights,\n", " shape=[num_examples, max_predictions_per_seq],\n", " dtype=tf.float32),\n", " \"next_sentence_labels\":\n", " tf.constant(\n", " all_next_sentence_labels,\n", " shape=[num_examples, 1],\n", " dtype=tf.int32),\n", " })\n", "\n", " d = d.batch(batch_size=batch_size, drop_remainder=False)\n", " return d\n", "\n", " return input_fn\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:35.671603Z", "start_time": "2018-11-16T10:02:35.626167Z" }, "code_folding": [ 64, 77 ] }, "outputs": [], "source": [ "def model_fn_builder(bert_config, init_checkpoint, learning_rate,\n", " num_train_steps, num_warmup_steps, use_tpu,\n", " use_one_hot_embeddings):\n", " \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n", "\n", " def model_fn(features, labels, mode, params): # pylint: disable=unused-argument\n", " \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n", "\n", " tf.logging.info(\"*** Features ***\")\n", " for name in sorted(features.keys()):\n", " tf.logging.info(\" name = %s, shape = %s\" % (name, features[name].shape))\n", "\n", " input_ids = features[\"input_ids\"]\n", " input_mask = features[\"input_mask\"]\n", " segment_ids = features[\"segment_ids\"]\n", " masked_lm_positions = features[\"masked_lm_positions\"]\n", " masked_lm_ids = features[\"masked_lm_ids\"]\n", " masked_lm_weights = features[\"masked_lm_weights\"]\n", " next_sentence_labels = features[\"next_sentence_labels\"]\n", "\n", " is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n", "\n", " model = tfm.BertModel(\n", " config=bert_config,\n", " is_training=is_training,\n", " input_ids=input_ids,\n", " input_mask=input_mask,\n", " token_type_ids=segment_ids,\n", " use_one_hot_embeddings=use_one_hot_embeddings)\n", "\n", " (masked_lm_loss,\n", " masked_lm_example_loss, masked_lm_log_probs) = rp.get_masked_lm_output(\n", " bert_config, model.get_sequence_output(), model.get_embedding_table(),\n", " masked_lm_positions, masked_lm_ids, masked_lm_weights)\n", "\n", " (next_sentence_loss, next_sentence_example_loss,\n", " next_sentence_log_probs) = rp.get_next_sentence_output(\n", " bert_config, model.get_pooled_output(), next_sentence_labels)\n", "\n", " total_loss = masked_lm_loss + next_sentence_loss\n", "\n", " tvars = tf.trainable_variables()\n", "\n", " initialized_variable_names = {}\n", " scaffold_fn = None\n", " if init_checkpoint:\n", " (assignment_map,\n", " initialized_variable_names) = tfm.get_assigment_map_from_checkpoint(\n", " tvars, init_checkpoint)\n", " if use_tpu:\n", "\n", " def tpu_scaffold():\n", " tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n", " return tf.train.Scaffold()\n", "\n", " scaffold_fn = tpu_scaffold\n", " else:\n", " tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n", "\n", " tf.logging.info(\"**** Trainable Variables ****\")\n", " for var in tvars:\n", " init_string = \"\"\n", " if var.name in initialized_variable_names:\n", " init_string = \", *INIT_FROM_CKPT*\"\n", " tf.logging.info(\" name = %s, shape = %s%s\", var.name, var.shape,\n", " init_string)\n", "\n", " output_spec = None\n", " if mode == tf.estimator.ModeKeys.TRAIN:\n", " masked_lm_positions = features[\"masked_lm_positions\"]\n", " masked_lm_ids = features[\"masked_lm_ids\"]\n", " masked_lm_weights = features[\"masked_lm_weights\"]\n", " next_sentence_labels = features[\"next_sentence_labels\"]\n", " train_op = optimization.create_optimizer(\n", " total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)\n", "\n", " output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n", " mode=mode,\n", " loss=total_loss,\n", " train_op=train_op,\n", " scaffold_fn=scaffold_fn)\n", " elif mode == tf.estimator.ModeKeys.EVAL:\n", " masked_lm_positions = features[\"masked_lm_positions\"]\n", " masked_lm_ids = features[\"masked_lm_ids\"]\n", " masked_lm_weights = features[\"masked_lm_weights\"]\n", " next_sentence_labels = features[\"next_sentence_labels\"]\n", "\n", " def metric_fn(masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids,\n", " masked_lm_weights, next_sentence_example_loss,\n", " next_sentence_log_probs, next_sentence_labels):\n", " \"\"\"Computes the loss and accuracy of the model.\"\"\"\n", " masked_lm_log_probs = tf.reshape(masked_lm_log_probs,\n", " [-1, masked_lm_log_probs.shape[-1]])\n", " masked_lm_predictions = tf.argmax(\n", " masked_lm_log_probs, axis=-1, output_type=tf.int32)\n", " masked_lm_example_loss = tf.reshape(masked_lm_example_loss, [-1])\n", " masked_lm_ids = tf.reshape(masked_lm_ids, [-1])\n", " masked_lm_weights = tf.reshape(masked_lm_weights, [-1])\n", " masked_lm_accuracy = tf.metrics.accuracy(\n", " labels=masked_lm_ids,\n", " predictions=masked_lm_predictions,\n", " weights=masked_lm_weights)\n", " masked_lm_mean_loss = tf.metrics.mean(\n", " values=masked_lm_example_loss, weights=masked_lm_weights)\n", "\n", " next_sentence_log_probs = tf.reshape(\n", " next_sentence_log_probs, [-1, next_sentence_log_probs.shape[-1]])\n", " next_sentence_predictions = tf.argmax(\n", " next_sentence_log_probs, axis=-1, output_type=tf.int32)\n", " next_sentence_labels = tf.reshape(next_sentence_labels, [-1])\n", " next_sentence_accuracy = tf.metrics.accuracy(\n", " labels=next_sentence_labels, predictions=next_sentence_predictions)\n", " next_sentence_mean_loss = tf.metrics.mean(\n", " values=next_sentence_example_loss)\n", "\n", " return {\n", " \"masked_lm_accuracy\": masked_lm_accuracy,\n", " \"masked_lm_loss\": masked_lm_mean_loss,\n", " \"next_sentence_accuracy\": next_sentence_accuracy,\n", " \"next_sentence_loss\": next_sentence_mean_loss,\n", " }\n", "\n", " eval_metrics = (metric_fn, [\n", " masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids,\n", " masked_lm_weights, next_sentence_example_loss,\n", " next_sentence_log_probs, next_sentence_labels\n", " ])\n", " output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n", " mode=mode,\n", " loss=total_loss,\n", " eval_metrics=eval_metrics,\n", " scaffold_fn=scaffold_fn)\n", " elif mode == tf.estimator.ModeKeys.PREDICT:\n", " masked_lm_log_probs = tf.reshape(masked_lm_log_probs,\n", " [-1, masked_lm_log_probs.shape[-1]])\n", " masked_lm_predictions = tf.argmax(\n", " masked_lm_log_probs, axis=-1, output_type=tf.int32)\n", "\n", " next_sentence_log_probs = tf.reshape(\n", " next_sentence_log_probs, [-1, next_sentence_log_probs.shape[-1]])\n", " next_sentence_predictions = tf.argmax(\n", " next_sentence_log_probs, axis=-1, output_type=tf.int32)\n", "\n", " masked_lm_predictions = tf.reshape(masked_lm_predictions,\n", " [1, masked_lm_positions.shape[-1]])\n", " next_sentence_predictions = tf.reshape(next_sentence_predictions,\n", " [1, 1])\n", "\n", " predictions = {\n", " \"masked_lm_predictions\": masked_lm_predictions,\n", " \"next_sentence_predictions\": next_sentence_predictions\n", " }\n", "\n", " output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n", " mode=mode, predictions=predictions, scaffold_fn=scaffold_fn)\n", " return output_spec\n", " else:\n", " raise ValueError(\"Only TRAIN, EVAL and PREDICT modes are supported: %s\" % (mode))\n", "\n", " return output_spec\n", "\n", " return model_fn" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:40.328700Z", "start_time": "2018-11-16T10:02:36.289676Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:Estimator's model_fn (.model_fn at 0x12a864ae8>) includes params argument, but params are not passed to Estimator.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - WARNING - tensorflow - Estimator's model_fn (.model_fn at 0x12a864ae8>) includes params argument, but params are not passed to Estimator.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - WARNING - tensorflow - Using temporary folder as model directory: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n", "graph_options {\n", " rewrite_options {\n", " meta_optimizer_iterations: ONE\n", " }\n", "}\n", ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': , '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=2, num_shards=1, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - Using config: {'_model_dir': '/var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n", "graph_options {\n", " rewrite_options {\n", " meta_optimizer_iterations: ONE\n", " }\n", "}\n", ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': , '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=2, num_shards=1, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:Setting TPUConfig.num_shards==1 is an unsupported behavior. Please fix as soon as possible (leaving num_shards as None.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - WARNING - tensorflow - Setting TPUConfig.num_shards==1 is an unsupported behavior. Please fix as soon as possible (leaving num_shards as None.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:_TPUContext: eval_on_tpu True\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - _TPUContext: eval_on_tpu True\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - WARNING - tensorflow - eval_on_tpu ignored because use_tpu is False.\n" ] } ], "source": [ "is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2\n", "run_config = tf.contrib.tpu.RunConfig(\n", " master=None,\n", " tpu_config=tf.contrib.tpu.TPUConfig(\n", " num_shards=1,\n", " per_host_input_for_training=is_per_host))\n", "\n", "model_fn = model_fn_builder(\n", " bert_config=bert_config,\n", " init_checkpoint=init_checkpoint,\n", " learning_rate=0,\n", " num_train_steps=1,\n", " num_warmup_steps=1,\n", " use_tpu=False,\n", " use_one_hot_embeddings=False)\n", "\n", "# If TPU is not available, this will fall back to normal Estimator on CPU\n", "# or GPU.\n", "estimator = tf.contrib.tpu.TPUEstimator(\n", " use_tpu=False,\n", " model_fn=model_fn,\n", " config=run_config,\n", " predict_batch_size=1)\n", "\n", "input_fn = input_fn_builder(\n", " features=features, seq_length=max_seq_length, max_predictions_per_seq=max_predictions_per_seq,\n", "tokenizer=tokenizer)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:46.596956Z", "start_time": "2018-11-16T10:02:40.331008Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Could not find trained model in model_dir: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d, running initialization to predict.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - Could not find trained model in model_dir: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmp4x8r3x3d, running initialization to predict.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Calling model_fn.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - Calling model_fn.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Running infer on CPU\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - Running infer on CPU\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:*** Features ***\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - *** Features ***\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = input_ids, shape = (?, 128)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = input_ids, shape = (?, 128)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = input_mask, shape = (?, 128)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = input_mask, shape = (?, 128)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = masked_lm_ids, shape = (?, 20)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = masked_lm_ids, shape = (?, 20)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = masked_lm_positions, shape = (?, 20)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = masked_lm_positions, shape = (?, 20)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = masked_lm_weights, shape = (?, 20)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = masked_lm_weights, shape = (?, 20)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = next_sentence_labels, shape = (?, 1)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = next_sentence_labels, shape = (?, 1)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = segment_ids, shape = (?, 128)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:40 - INFO - tensorflow - name = segment_ids, shape = (?, 128)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:**** Trainable Variables ****\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - **** Trainable Variables ****\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/predictions/output_bias:0, shape = (30522,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow: name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Done calling model_fn.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:43 - INFO - tensorflow - Done calling model_fn.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Graph was finalized.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:44 - INFO - tensorflow - Graph was finalized.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Running local_init_op.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:45 - INFO - tensorflow - Running local_init_op.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Done running local_init_op.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:45 - INFO - tensorflow - Done running local_init_op.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:prediction_loop marked as finished\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:46 - INFO - tensorflow - prediction_loop marked as finished\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:prediction_loop marked as finished\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:02:46 - INFO - tensorflow - prediction_loop marked as finished\n" ] } ], "source": [ "tensorflow_all_out = []\n", "for result in estimator.predict(input_fn, yield_single_examples=True):\n", " tensorflow_all_out.append(result)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:46.634304Z", "start_time": "2018-11-16T10:02:46.598800Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1\n", "2\n", "dict_keys(['masked_lm_predictions', 'next_sentence_predictions'])\n", "masked_lm_predictions [27227 1010 1010 1010 1010 1010 1010 1010 1010 1010 1010 1010\n", " 1010 1010 1010 1010 1010 1010 1010 1010]\n", "predicted token ['henson', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',', ',']\n" ] } ], "source": [ "print(len(tensorflow_all_out))\n", "print(len(tensorflow_all_out[0]))\n", "print(tensorflow_all_out[0].keys())\n", "print(\"masked_lm_predictions\", tensorflow_all_out[0]['masked_lm_predictions'])\n", "print(\"predicted token\", tokenizer.convert_ids_to_tokens(tensorflow_all_out[0]['masked_lm_predictions']))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:02:46.671229Z", "start_time": "2018-11-16T10:02:46.637102Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensorflow_output: ['henson']\n" ] } ], "source": [ "tensorflow_outputs = tokenizer.convert_ids_to_tokens(tensorflow_all_out[0]['masked_lm_predictions'])[:len(masked_lm_positions)]\n", "print(\"tensorflow_output:\", tensorflow_outputs)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2/ PyTorch code" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:03.556557Z", "start_time": "2018-11-16T10:03:03.519654Z" } }, "outputs": [], "source": [ "from examples import extract_features\n", "from examples.extract_features import *" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:03.952710Z", "start_time": "2018-11-16T10:03:03.921917Z" } }, "outputs": [], "source": [ "init_checkpoint_pt = \"../google_models/uncased_L-12_H-768_A-12/pytorch_model.bin\"" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:12.307673Z", "start_time": "2018-11-16T10:03:04.439317Z" }, "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "11/16/2018 11:03:05 - INFO - pytorch_transformers.modeling_bert - loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /Users/thomaswolf/.pytorch_transformers/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n", "11/16/2018 11:03:05 - INFO - pytorch_transformers.modeling_bert - extracting archive file /Users/thomaswolf/.pytorch_transformers/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmpaqgsm566\n", "11/16/2018 11:03:08 - INFO - pytorch_transformers.modeling_bert - Model config {\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"max_position_embeddings\": 512,\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"type_vocab_size\": 2,\n", " \"vocab_size\": 30522\n", "}\n", "\n" ] }, { "data": { "text/plain": [ "BertForPreTraining(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(30522, 768)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (cls): BertPreTrainingHeads(\n", " (predictions): BertLMPredictionHead(\n", " (transform): BertPredictionHeadTransform(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " )\n", " (decoder): Linear(in_features=768, out_features=30522, bias=False)\n", " )\n", " (seq_relationship): Linear(in_features=768, out_features=2, bias=True)\n", " )\n", ")" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "device = torch.device(\"cpu\")\n", "model = ppb.BertForPreTraining.from_pretrained('bert-base-uncased')\n", "model.to(device)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:12.351625Z", "start_time": "2018-11-16T10:03:12.310736Z" }, "code_folding": [] }, "outputs": [ { "data": { "text/plain": [ "BertForPreTraining(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(30522, 768)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (1): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (2): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (3): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (4): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (5): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (6): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (7): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (8): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (9): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (10): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (11): BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " (dropout): Dropout(p=0.1)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (cls): BertPreTrainingHeads(\n", " (predictions): BertLMPredictionHead(\n", " (transform): BertPredictionHeadTransform(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): BertLayerNorm()\n", " )\n", " (decoder): Linear(in_features=768, out_features=30522, bias=False)\n", " )\n", " (seq_relationship): Linear(in_features=768, out_features=2, bias=True)\n", " )\n", ")" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n", "all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)\n", "all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)\n", "all_masked_lm_positions = torch.tensor([f.masked_lm_positions for f in features], dtype=torch.long)\n", "\n", "eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_masked_lm_positions)\n", "eval_sampler = SequentialSampler(eval_data)\n", "eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=1)\n", "\n", "model.eval()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:12.792741Z", "start_time": "2018-11-16T10:03:12.354253Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[ 2040, 2001, 3958, 27227, 1029, 3958, 103, 2001, 1037, 13997,\n", " 11510, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0]])\n", "tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0]])\n", "tensor([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", " 0, 0, 0, 0, 0, 0, 0, 0]])\n", "(1, 20, 30522)\n", "[27227, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010, 1010]\n" ] } ], "source": [ "import numpy as np\n", "pytorch_all_out = []\n", "for input_ids, input_mask, segment_ids, tensor_masked_lm_positions in eval_dataloader:\n", " print(input_ids)\n", " print(input_mask)\n", " print(segment_ids)\n", " input_ids = input_ids.to(device)\n", " input_mask = input_mask.to(device)\n", " segment_ids = segment_ids.to(device)\n", "\n", " prediction_scores, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)\n", " prediction_scores = prediction_scores[0, tensor_masked_lm_positions].detach().cpu().numpy()\n", " print(prediction_scores.shape)\n", " masked_lm_predictions = np.argmax(prediction_scores, axis=-1).squeeze().tolist()\n", " print(masked_lm_predictions)\n", " pytorch_all_out.append(masked_lm_predictions)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2018-11-16T10:03:12.828439Z", "start_time": "2018-11-16T10:03:12.795420Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pytorch_output: ['henson']\n", "tensorflow_output: ['henson']\n" ] } ], "source": [ "pytorch_outputs = tokenizer.convert_ids_to_tokens(pytorch_all_out[0])[:len(masked_lm_positions)]\n", "print(\"pytorch_output:\", pytorch_outputs)\n", "print(\"tensorflow_output:\", tensorflow_outputs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "hide_input": false, "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.7" }, "toc": { "colors": { "hover_highlight": "#DAA520", "running_highlight": "#FF0000", "selected_highlight": "#FFD700" }, "moveMenuLeft": true, "nav_menu": { "height": "48px", "width": "252px" }, "navigate_menu": true, "number_sections": true, "sideBar": true, "threshold": 4, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }