astroBERT model ner model all_labeled_data_run01 checkpoint-169600

.ipynb_checkpoints/README-checkpoint.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

Tutorials/.ipynb_checkpoints/0_Embeddings-checkpoint.ipynb

@@ -0,0 +1,295 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "274e6135-2d97-4244-9183-65bcb1d24c80",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use the trained astroBERT model to generate embedings of text\n",
+    "# to be used for downstream tasks"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2cc88ed3-6f52-49a2-99c0-344387758ab5",
+   "metadata": {},
+   "source": [
+    "# Tutorial 0: Loading astroBERT to produce text embeddings\n",
+    "This tutorial will show you how to load astroBERT and produce text embeddings that can be used on downstream tasks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9e65c041-9d66-4fb1-96b9-4937000da02e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 1 - load models and tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "67d99e96-c532-49ef-8542-a48eef818956",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-10-20 16:07:24.705905: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "00e1d48e-9898-44ef-b00e-43e3ab7fed7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the model path can either be the name of the Huggingface repository\n",
+    "remote_model_path = 'adsabs/astroBERT'\n",
+    "# or the local path to the directory containing model weight and tokenizer vocab\n",
+    "local_model_path = '../'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "9bcc6009-6009-463f-a7da-f010c5fae27e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make sure you load the tokenier with do_lower_case=False\n",
+    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
+    "                                                    use_auth_token=True,\n",
+    "                                                    add_special_tokens=True,\n",
+    "                                                    do_lower_case=False,\n",
+    "                                                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dbd144f0-6038-4917-94b0-aea9da72cac5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "PreTrainedTokenizerFast(name_or_path='adsabs/astroBERT', vocab_size=30000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "astroBERT_tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "dd9a9257-cbe4-4908-a9f4-8e1431dc375a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
+      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "# automodels: defaults to BertModel\n",
+    "# it's normal to get warnings as a BertModel will not load the weights used for PreTraining\n",
+    "astroBERT_automodel = AutoModel.from_pretrained(remote_model_path, \n",
+    "                                                use_auth_token=True,\n",
+    "                                               )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "572ddd38-a0dc-4583-a5a6-c4f3b2cb2553",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 2 - make some inference, the outputs are the embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "32fc0b97-4a2d-42ab-aa83-f5d8b39672b1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([3, 54])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# list of strings for which we want embeddings\n",
+    "strings = ['The Chandra X-ray Observatory (CXO), previously known as the Advanced X-ray Astrophysics Facility (AXAF), is a Flagship-class space telescope launched aboard the Space Shuttle Columbia during STS-93 by NASA on July 23, 1999.',\n",
+    "           'Independent lines of evidence from Type Ia supernovae and the CMB imply that the universe today is dominated by a mysterious form of energy known as dark energy, which appears to homogeneously permeate all of space.',\n",
+    "           'This work has been developed in the framework of the ‘Darklight’ programme, supported by the European Research Council through an Advanced Research Grant to LG (Project # 291521).'\n",
+    "          ]\n",
+    "\n",
+    "# tokenizer the strings, with padding (needed to process multiple strings efficiently)\n",
+    "inputs = astroBERT_tokenizer(strings, \n",
+    "                             padding=True, \n",
+    "                             return_tensors='pt'\n",
+    "                            )\n",
+    "\n",
+    "# check the shape of the inputs\n",
+    "print(inputs['input_ids'].shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8b7c9456-573a-48e7-9bc2-839fcc25631d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pass the inputs through astroBERT\n",
+    "import torch\n",
+    "# no need for gradients, since we are only doing inference\n",
+    "with torch.no_grad():\n",
+    "    output = astroBERT_automodel(**inputs, \n",
+    "                                 output_hidden_states=False\n",
+    "                                )  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "116de57a-bb31-48d7-9556-64e01a16d56f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([3, 54, 768])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# BertModel outputs two tensors: last_hidden_state (our embeddings) and pooler_output (to be discarded as it's not meaningful)\n",
+    "# see https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel.forward\n",
+    "# embeddings will have shape = (# of strings, size of tokenized strings(padded), 768 (BERT embedding size))\n",
+    "embeddings = output[0]\n",
+    "print(embeddings.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "38e45291-6fd7-48cf-83df-e1cc5c8a699f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[ 0.5546,  0.9121,  0.6550,  ..., -0.1925,  0.7077, -0.2405],\n",
+      "        [ 0.6252,  0.3175,  1.0899,  ...,  0.0576,  0.0529,  0.0603],\n",
+      "        [ 0.1803, -0.4567,  1.2688,  ...,  0.6026, -0.5718, -0.2060],\n",
+      "        ...,\n",
+      "        [-0.4397, -0.5334,  1.1682,  ...,  0.9541,  0.4046, -0.4756],\n",
+      "        [-0.3911,  0.7793,  0.2432,  ...,  0.2268, -1.0489, -1.4864],\n",
+      "        [-0.4529, -0.7346,  0.0675,  ..., -0.3246, -0.2333, -0.6154]])\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(embeddings[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "26acf89f-b7fc-4872-ac81-0ee65030b465",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# If you wish to use the hidden states as additional embeddings, you can use output_hidden_states=True\n",
+    "\n",
+    "# no need for gradients, since we are only doing inference\n",
+    "with torch.no_grad():\n",
+    "    output = astroBERT_automodel(**inputs, \n",
+    "                                 output_hidden_states=True\n",
+    "                                )  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "a54314e9-5dcb-4c10-b0d2-219a93c7d16e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "13\n",
+      "torch.Size([3, 54, 768])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# This will produce 13 embeddings, one for each hidden layer\n",
+    "embeddings = output[2]\n",
+    "print(len(embeddings))\n",
+    "print(embeddings[0].shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76765dcb-8035-44b2-a5a3-db181b561095",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Tutorials/.ipynb_checkpoints/1_Fill-Mask-checkpoint.ipynb

@@ -0,0 +1,425 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "33df4373-a37b-4fd0-bc67-c297812871e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use the trained astroBERT model with the fill-mask pipeline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "164ee9bd-27f9-40a4-8461-3ce12fc928b0",
+   "metadata": {},
+   "source": [
+    "# Tutorial 1: using astroBERT with the fill-mask pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "59429414-f07e-45e5-8825-6fc6a8d26653",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 1 - load models and tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "db8ee724-6a2a-4ea5-820e-5e2aa0a0f622",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-10-17 21:17:27.369794: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, BertForMaskedLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9a98fb63-0793-4684-a202-931cad17c7ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the model path can either be the name of the Huggingface repository\n",
+    "remote_model_path = 'adsabs/astroBERT'\n",
+    "# or the local path to the directory containing model weight and tokenizer vocab\n",
+    "local_model_path = '../'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "25fedd16-283b-4817-9b19-2a5ff1c5ba88",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make sure you load the tokenier with do_lower_case=False\n",
+    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
+    "                                                    use_auth_token=True,\n",
+    "                                                    add_special_tokens=False,\n",
+    "                                                    do_lower_case=False,\n",
+    "                                                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fb10db03-a5f0-44f7-8d41-0285f898a90d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6e0bf5ee71b4986a682adb43e994ede",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/666 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
+      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "astroBERT_automodel_for_mlm = BertForMaskedLM.from_pretrained(pretrained_model_name_or_path=remote_model_path, \n",
+    "                                                use_auth_token=True,\n",
+    "                                               )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e8b9b073-3876-4d0b-b8b2-e46fa25c76f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# for pipeline to work you have to ensure that the model returns a dict\n",
+    "astroBERT_automodel_for_mlm.config.return_dict=True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "94338f6f-3467-4696-bf7d-f41a12eb889d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import FillMaskPipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "7b980d9f-4d86-4b54-9324-d57dd9b4b64f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "astroBERT_pipeline = FillMaskPipeline(model=astroBERT_automodel_for_mlm,\n",
+    "                                      tokenizer=astroBERT_tokenizer,\n",
+    "                                      task='fill-mask',\n",
+    "                                     )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5cb4d27b-ee3c-4ac7-ace2-4cc57ea9ce7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clean_sentences = ['M67 is one of the most studied open clusters.',\n",
+    "'A solar twin is a star with atmospheric parameters and chemical composition very similar to our Sun.',\n",
+    "'The dynamical evolution of planets close to their star is affected by tidal effects',\n",
+    "'The Kepler satellite collected high-precision long-term and continuous light curves for more than 100,000 solar-type stars',\n",
+    "'The Local Group is composed of the Milky Way, the Andromeda Galaxy, and numerous smaller satellite galaxies.',\n",
+    "'Cepheid variables are used to determine the distances to galaxies in the local universe.',\n",
+    "'Jets are created and sustained by accretion of matter onto a compact massive object.',\n",
+    "'A single star of one solar mass will evolve into a white dwarf.',\n",
+    "'The Very Large Array observes the sky at radio wavelengths.',\n",
+    "'Elements heavier than iron are generated in supernovae explosions.',\n",
+    "'Spitzer was the first spacecraft to fly in an Earth-trailing orbit.',\n",
+    "'Galaxy mergers can occur when two (or more) galaxies collide',\n",
+    "'Dark matter is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
+    "'The Local Group of galaxies is pulled toward The Great Attractor.',\n",
+    "'The Moon is the only satellite of the Earth.',\n",
+    "'Galaxies are categorized according to their visual morphology as elliptical, spiral, or irregular.',\n",
+    "'Stars are made mostly of hydrogen.',\n",
+    "'Comet tails are created as comets approach the Sun.',\n",
+    "'Pluto is a dwarf planet in the Kuiper Belt.',\n",
+    "'The Milky Way has a supermassive black hole, Sagittarius A*, at its center.',\n",
+    "'Andromeda is the nearest large galaxy to the Milky Way and is roughly its equal in mass.',\n",
+    "'The interstellar medium is the gas and dust between stars.',\n",
+    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the universe.',\n",
+    "'The Large and Small Magellanic Clouds are irregular dwarf galaxies and are two satellite galaxies of the Milky Way.']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "9f3a6fdc-182f-4edb-8ef4-7e4253c2d4db",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "masked_sentences =  ['M67 is one of the most studied [MASK] clusters.',\n",
+    "'A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.',\n",
+    "'The dynamical evolution of planets close to their star is affected by [MASK] effects',\n",
+    "'The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars',\n",
+    "'The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.',\n",
+    "'Cepheid variables are used to determine the [MASK] to galaxies in the local universe.',\n",
+    "'Jets are created and sustained by [MASK] of matter onto a compact massive object.',\n",
+    "'A single star of one solar mass will evolve into a [MASK] dwarf.',\n",
+    "'The Very Large Array observes the sky at [MASK] wavelengths.',\n",
+    "'Elements heavier than [MASK] are generated in supernovae explosions.',\n",
+    "'Spitzer was the first [MASK] to fly in an Earth-trailing orbit.',\n",
+    "'Galaxy [MASK] can occur when two (or more) galaxies collide',\n",
+    "'Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
+    "'The Local Group of galaxies is pulled toward The Great [MASK] .',\n",
+    "'The Moon is the only [MASK] of the Earth.',\n",
+    "'Galaxies are categorized according to their visual morphology as [MASK] , spiral, or irregular.',\n",
+    "'Stars are made mostly of [MASK] .',\n",
+    "'Comet tails are created as comets approach the [MASK] .',\n",
+    "'Pluto is a dwarf [MASK] in the Kuiper Belt.',\n",
+    "'The Milky Way has a [MASK] black hole, Sagittarius A*, at its center.',\n",
+    "'Andromeda is the nearest large [MASK] to the Milky Way and is roughly its equal in mass.',\n",
+    "'The [MASK] medium is the gas and dust between stars.',\n",
+    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK] .',\n",
+    "'The Large and Small Magellanic Clouds are irregular [MASK] galaxies and are two satellite galaxies of the Milky Way.',\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d4c729ad-89f4-4e70-b433-a65b6035c10b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "masked_words = [x for s1,s2 in zip(clean_sentences, masked_sentences) \n",
+    "              for x,y in zip(s1.split(), s2.split()) if y=='[MASK]']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "2a07a641-61a7-42dd-b70e-62eb97ad4e4b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = astroBERT_pipeline(inputs=masked_sentences, \n",
+    "                             top_k=3\n",
+    "                            )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "ec2880d9-a8ad-4919-ab5b-732f3bcc21ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "M67 is one of the most studied [MASK] clusters.\n",
+      "original: open\n",
+      "\t  open 0.87\n",
+      "\t  globular 0.07\n",
+      "\t  star 0.03\n",
+      "\n",
+      "A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.\n",
+      "original: atmospheric\n",
+      "\t  fundamental 0.56\n",
+      "\t  physical 0.25\n",
+      "\t  stellar 0.05\n",
+      "\n",
+      "The dynamical evolution of planets close to their star is affected by [MASK] effects\n",
+      "original: tidal\n",
+      "\t  tidal 0.07\n",
+      "\t  electromagnetic 0.05\n",
+      "\t  electrostatic 0.04\n",
+      "\n",
+      "The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars\n",
+      "original: curves\n",
+      "\t  curves 0.43\n",
+      "\t  ##s 0.04\n",
+      "\t  conditions 0.04\n",
+      "\n",
+      "The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.\n",
+      "original: Andromeda\n",
+      "\t  Andromeda 0.99\n",
+      "\t  M31 0.00\n",
+      "\t  Sagittarius 0.00\n",
+      "\n",
+      "Cepheid variables are used to determine the [MASK] to galaxies in the local universe.\n",
+      "original: distances\n",
+      "\t  distances 0.79\n",
+      "\t  distance 0.21\n",
+      "\t  redshifts 0.00\n",
+      "\n",
+      "Jets are created and sustained by [MASK] of matter onto a compact massive object.\n",
+      "original: accretion\n",
+      "\t  accretion 0.79\n",
+      "\t  infall 0.13\n",
+      "\t  fall 0.02\n",
+      "\n",
+      "A single star of one solar mass will evolve into a [MASK] dwarf.\n",
+      "original: white\n",
+      "\t  white 0.77\n",
+      "\t  brown 0.19\n",
+      "\t  red 0.02\n",
+      "\n",
+      "The Very Large Array observes the sky at [MASK] wavelengths.\n",
+      "original: radio\n",
+      "\t  radio 0.29\n",
+      "\t  centimeter 0.10\n",
+      "\t  all 0.09\n",
+      "\n",
+      "Elements heavier than [MASK] are generated in supernovae explosions.\n",
+      "original: iron\n",
+      "\t  iron 0.34\n",
+      "\t  helium 0.16\n",
+      "\t  oxygen 0.07\n",
+      "\n",
+      "Spitzer was the first [MASK] to fly in an Earth-trailing orbit.\n",
+      "original: spacecraft\n",
+      "\t  satellite 0.42\n",
+      "\t  spacecraft 0.20\n",
+      "\t  observatory 0.16\n",
+      "\n",
+      "Galaxy [MASK] can occur when two (or more) galaxies collide\n",
+      "original: mergers\n",
+      "\t  . 0.26\n",
+      "\t  A 0.05\n",
+      "\t  1 0.04\n",
+      "\n",
+      "Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.\n",
+      "original: matter\n",
+      "\t  energy 0.64\n",
+      "\t  Energy 0.24\n",
+      "\t  matter 0.10\n",
+      "\n",
+      "The Local Group of galaxies is pulled toward The Great [MASK] .\n",
+      "original: Attractor.\n",
+      "\t  Wall 0.96\n",
+      "\t  East 0.01\n",
+      "\t  Planet 0.00\n",
+      "\n",
+      "The Moon is the only [MASK] of the Earth.\n",
+      "original: satellite\n",
+      "\t  satellite 0.38\n",
+      "\t  moon 0.31\n",
+      "\t  constituent 0.07\n",
+      "\n",
+      "Galaxies are categorized according to their visual morphology as [MASK] , spiral, or irregular.\n",
+      "original: elliptical,\n",
+      "\t  elliptical 0.92\n",
+      "\t  spheroidal 0.02\n",
+      "\t  irregular 0.01\n",
+      "\n",
+      "Stars are made mostly of [MASK] .\n",
+      "original: hydrogen.\n",
+      "\t  hydrogen 0.20\n",
+      "\t  helium 0.14\n",
+      "\t  carbon 0.12\n",
+      "\n",
+      "Comet tails are created as comets approach the [MASK] .\n",
+      "original: Sun.\n",
+      "\t  Sun 0.45\n",
+      "\t  sun 0.23\n",
+      "\t  Earth 0.19\n",
+      "\n",
+      "Pluto is a dwarf [MASK] in the Kuiper Belt.\n",
+      "original: planet\n",
+      "\t  planet 0.96\n",
+      "\t  satellite 0.02\n",
+      "\t  nova 0.00\n",
+      "\n",
+      "The Milky Way has a [MASK] black hole, Sagittarius A*, at its center.\n",
+      "original: supermassive\n",
+      "\t  supermassive 0.80\n",
+      "\t  massive 0.17\n",
+      "\t  stellar 0.00\n",
+      "\n",
+      "Andromeda is the nearest large [MASK] to the Milky Way and is roughly its equal in mass.\n",
+      "original: galaxy\n",
+      "\t  galaxy 0.68\n",
+      "\t  spiral 0.12\n",
+      "\t  satellite 0.09\n",
+      "\n",
+      "The [MASK] medium is the gas and dust between stars.\n",
+      "original: interstellar\n",
+      "\t  interstellar 0.87\n",
+      "\t  interplanetary 0.05\n",
+      "\t  intracluster 0.03\n",
+      "\n",
+      "The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK] .\n",
+      "original: universe.\n",
+      "\t  universe 0.45\n",
+      "\t  Universe 0.26\n",
+      "\t  expansion 0.09\n",
+      "\n",
+      "The Large and Small Magellanic Clouds are irregular [MASK] galaxies and are two satellite galaxies of the Milky Way.\n",
+      "original: dwarf\n",
+      "\t  dwarf 0.68\n",
+      "\t  satellite 0.13\n",
+      "\t  Magellanic 0.08\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "for w, s, rs in zip(masked_words, masked_sentences, results):\n",
+    "    print(s)\n",
+    "    print('original: {}'.format(w))\n",
+    "    for r in rs:\n",
+    "        print('\\t  {} {:0.2f}'.format(r['token_str'], r['score']))\n",
+    "    print()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Tutorials/0_Embeddings.ipynb

@@ -40,7 +40,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2022-10-19 10:05:02.842926: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+      "2022-10-20 16:07:24.705905: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
      ]
     }
    ],
@@ -107,7 +107,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
       "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
       "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
      ]

config.json

@@ -1,7 +1,7 @@
 {
-  "_name_or_path": "adsabs/astroBERT",
+  "_name_or_path": "../astroBERT-Tasks/Finetuning_1_NER/trained-models/NER_astroBERT_all_labeled_data_run01/checkpoint-169600/",
   "architectures": [
-    "BertForPreTraining"
+    "BertForTokenClassification"
   ],
   "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
@@ -9,8 +9,138 @@
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2",
+    "3": "LABEL_3",
+    "4": "LABEL_4",
+    "5": "LABEL_5",
+    "6": "LABEL_6",
+    "7": "LABEL_7",
+    "8": "LABEL_8",
+    "9": "LABEL_9",
+    "10": "LABEL_10",
+    "11": "LABEL_11",
+    "12": "LABEL_12",
+    "13": "LABEL_13",
+    "14": "LABEL_14",
+    "15": "LABEL_15",
+    "16": "LABEL_16",
+    "17": "LABEL_17",
+    "18": "LABEL_18",
+    "19": "LABEL_19",
+    "20": "LABEL_20",
+    "21": "LABEL_21",
+    "22": "LABEL_22",
+    "23": "LABEL_23",
+    "24": "LABEL_24",
+    "25": "LABEL_25",
+    "26": "LABEL_26",
+    "27": "LABEL_27",
+    "28": "LABEL_28",
+    "29": "LABEL_29",
+    "30": "LABEL_30",
+    "31": "LABEL_31",
+    "32": "LABEL_32",
+    "33": "LABEL_33",
+    "34": "LABEL_34",
+    "35": "LABEL_35",
+    "36": "LABEL_36",
+    "37": "LABEL_37",
+    "38": "LABEL_38",
+    "39": "LABEL_39",
+    "40": "LABEL_40",
+    "41": "LABEL_41",
+    "42": "LABEL_42",
+    "43": "LABEL_43",
+    "44": "LABEL_44",
+    "45": "LABEL_45",
+    "46": "LABEL_46",
+    "47": "LABEL_47",
+    "48": "LABEL_48",
+    "49": "LABEL_49",
+    "50": "LABEL_50",
+    "51": "LABEL_51",
+    "52": "LABEL_52",
+    "53": "LABEL_53",
+    "54": "LABEL_54",
+    "55": "LABEL_55",
+    "56": "LABEL_56",
+    "57": "LABEL_57",
+    "58": "LABEL_58",
+    "59": "LABEL_59",
+    "60": "LABEL_60",
+    "61": "LABEL_61",
+    "62": "LABEL_62"
+  },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_11": 11,
+    "LABEL_12": 12,
+    "LABEL_13": 13,
+    "LABEL_14": 14,
+    "LABEL_15": 15,
+    "LABEL_16": 16,
+    "LABEL_17": 17,
+    "LABEL_18": 18,
+    "LABEL_19": 19,
+    "LABEL_2": 2,
+    "LABEL_20": 20,
+    "LABEL_21": 21,
+    "LABEL_22": 22,
+    "LABEL_23": 23,
+    "LABEL_24": 24,
+    "LABEL_25": 25,
+    "LABEL_26": 26,
+    "LABEL_27": 27,
+    "LABEL_28": 28,
+    "LABEL_29": 29,
+    "LABEL_3": 3,
+    "LABEL_30": 30,
+    "LABEL_31": 31,
+    "LABEL_32": 32,
+    "LABEL_33": 33,
+    "LABEL_34": 34,
+    "LABEL_35": 35,
+    "LABEL_36": 36,
+    "LABEL_37": 37,
+    "LABEL_38": 38,
+    "LABEL_39": 39,
+    "LABEL_4": 4,
+    "LABEL_40": 40,
+    "LABEL_41": 41,
+    "LABEL_42": 42,
+    "LABEL_43": 43,
+    "LABEL_44": 44,
+    "LABEL_45": 45,
+    "LABEL_46": 46,
+    "LABEL_47": 47,
+    "LABEL_48": 48,
+    "LABEL_49": 49,
+    "LABEL_5": 5,
+    "LABEL_50": 50,
+    "LABEL_51": 51,
+    "LABEL_52": 52,
+    "LABEL_53": 53,
+    "LABEL_54": 54,
+    "LABEL_55": 55,
+    "LABEL_56": 56,
+    "LABEL_57": 57,
+    "LABEL_58": 58,
+    "LABEL_59": 59,
+    "LABEL_6": 6,
+    "LABEL_60": 60,
+    "LABEL_61": 61,
+    "LABEL_62": 62,
+    "LABEL_7": 7,
+    "LABEL_8": 8,
+    "LABEL_9": 9
+  },
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
   "model_type": "bert",
@@ -18,6 +148,7 @@
   "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
+  "return_dict": false,
   "torch_dtype": "float32",
   "transformers_version": "4.17.0",
   "type_vocab_size": 2,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3268b70d3529eb896bf43c0c8fa933f6282b82fa196e2659ab116d49fbbca6ac
-size 438904355
+oid sha256:069bc02d7c193d06fc9d23ce25f59129ff2c6724d424d60d600343e52f9e12f7
+size 434238321