# Use the trained astroBERT model to generate embedings of text
+# to be used for downstream tasks
+This tutorial will show you how to load astroBERT and produce text embeddings that can be used on downstream tasks.
+ +# 1 - load models and tokenizer
+from transformers import AutoTokenizer, AutoModel
+2022-10-17 12:10:19.355203: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0 ++
# the model path can either be the name of the Huggingface repository
+remote_model_path = 'adsabs/astroBERT'
+# or the local path to the directory containing model weight and tokenizer vocab
+local_model_path = '../'
+# make sure you load the tokenier with do_lower_case=False
+astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,
+ use_auth_token=True,
+ add_special_tokens=True,
+ do_lower_case=False,
+ )
+astroBERT_tokenizer
+PreTrainedTokenizerFast(name_or_path='adsabs/astroBERT', vocab_size=30000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})
+# automodels: defaults to BertModel
+# it's normal to get warnings as a BertModel will not load the weights used for PreTraining
+astroBERT_automodel = AutoModel.from_pretrained(remote_model_path,
+ use_auth_token=True,
+ )
+Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight'] +- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). +- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). ++
# 2 - make some inference, the outputs are the embeddings
+# list of strings for which we want embeddings
+strings = ['The Chandra X-ray Observatory (CXO), previously known as the Advanced X-ray Astrophysics Facility (AXAF), is a Flagship-class space telescope launched aboard the Space Shuttle Columbia during STS-93 by NASA on July 23, 1999.',
+ 'Independent lines of evidence from Type Ia supernovae and the CMB imply that the universe today is dominated by a mysterious form of energy known as dark energy, which appears to homogeneously permeate all of space.',
+ 'This work has been developed in the framework of the ‘Darklight’ programme, supported by the European Research Council through an Advanced Research Grant to LG (Project # 291521).'
+ ]
+
+# tokenizer the strings, with padding (needed to process multiple strings efficiently)
+inputs = astroBERT_tokenizer(strings,
+ padding=True,
+ return_tensors='pt'
+ )
+
+# check the shape of the inputs
+print(inputs['input_ids'].shape)
+torch.Size([3, 54]) ++
# pass the inputs through astroBERT
+import torch
+# no need for gradients, since we are only doing inference
+with torch.no_grad():
+ output = astroBERT_automodel(**inputs,
+ output_hidden_states=False
+ )
+# BertModel outputs two tensors: last_hidden_state (our embeddings) and pooler_output (to be discarded as it's not meaningful)
+# see https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel.forward
+# embeddings will have shape = (# of strings, size of tokenized strings(padded), 768 (BERT embedding size))
+embeddings = output[0]
+print(embeddings.shape)
+torch.Size([3, 54, 768]) ++
print(embeddings[0])
+tensor([[ 0.5546, 0.9121, 0.6550, ..., -0.1925, 0.7077, -0.2405], + [ 0.6252, 0.3175, 1.0899, ..., 0.0576, 0.0529, 0.0603], + [ 0.1803, -0.4567, 1.2688, ..., 0.6026, -0.5718, -0.2060], + ..., + [-0.4397, -0.5334, 1.1682, ..., 0.9541, 0.4046, -0.4756], + [-0.3911, 0.7793, 0.2432, ..., 0.2268, -1.0489, -1.4864], + [-0.4529, -0.7346, 0.0675, ..., -0.3246, -0.2333, -0.6154]]) ++
+